Diff of the two buildlogs: -- --- b1/build.log 2024-10-21 16:15:24.676847037 +0000 +++ b2/build.log 2024-10-21 17:49:33.684531297 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sun Nov 23 10:18:47 -12 2025 -I: pbuilder-time-stamp: 1763936327 +I: Current time: Tue Oct 22 06:15:27 +14 2024 +I: pbuilder-time-stamp: 1729527327 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/trixie-reproducible-base.tgz] I: copying local configuration @@ -45,52 +45,84 @@ dpkg-source: info: applying fix-version.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/2912599/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/D01_modify_environment starting +debug: Running on ionos1-amd64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Oct 21 16:15 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='amd64' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=42 ' - DISTRIBUTION='trixie' - HOME='/root' - HOST_ARCH='amd64' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="32" [3]="1" [4]="release" [5]="x86_64-pc-linux-gnu") + BASH_VERSION='5.2.32(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=amd64 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=20 ' + DIRSTACK=() + DISTRIBUTION=trixie + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=x86_64 + HOST_ARCH=amd64 IFS=' ' - INVOCATION_ID='a2263ed9979d43c6b666b5ed536b25e1' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='2912599' - PS1='# ' - PS2='> ' + INVOCATION_ID=350a2f94d4cf4409830d8be6f1095b76 + LANG=C + LANGUAGE=et_EE:et + LC_ALL=C + MACHTYPE=x86_64-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=1538210 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.S3dTZzvM/pbuilderrc_IBvX --distribution trixie --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.S3dTZzvM/b1 --logfile b1/build.log dask.distributed_2024.5.2+ds.1-7.dsc' - SUDO_GID='111' - SUDO_UID='106' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://213.165.73.152:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.S3dTZzvM/pbuilderrc_3Cpl --distribution trixie --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.S3dTZzvM/b2 --logfile b2/build.log dask.distributed_2024.5.2+ds.1-7.dsc' + SUDO_GID=110 + SUDO_UID=105 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://46.16.76.132:3128 I: uname -a - Linux ionos15-amd64 6.10.11+bpo-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.10.11-1~bpo12+1 (2024-10-03) x86_64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-26-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.112-1 (2024-09-30) x86_64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 Aug 4 2024 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/2912599/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Aug 4 21:30 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -444,7 +476,7 @@ Get: 230 http://deb.debian.org/debian trixie/main amd64 python3-zict all 3.0.0-2 [29.7 kB] Get: 231 http://deb.debian.org/debian trixie/main amd64 tzdata-legacy all 2024a-4 [170 kB] Get: 232 http://deb.debian.org/debian trixie/main amd64 uglifyjs all 3.17.4-2 [12.1 kB] -Fetched 136 MB in 8s (17.8 MB/s) +Fetched 136 MB in 6s (22.1 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package fonts-lato. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19799 files and directories currently installed.) @@ -1189,8 +1221,8 @@ Setting up tzdata (2024a-4) ... Current default time zone: 'Etc/UTC' -Local time is now: Sun Nov 23 22:19:53 UTC 2025. -Universal Time is now: Sun Nov 23 22:19:53 UTC 2025. +Local time is now: Mon Oct 21 16:19:20 UTC 2024. +Universal Time is now: Mon Oct 21 16:19:20 UTC 2024. Run 'dpkg-reconfigure tzdata' if you wish to change it. Setting up libpgm-5.3-0t64:amd64 (5.3.128~dfsg-2.1) ... @@ -1414,7 +1446,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/dask.distributed-2024.5.2+ds.1/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../dask.distributed_2024.5.2+ds.1-7_source.changes +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for trixie +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/dask.distributed-2024.5.2+ds.1/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../dask.distributed_2024.5.2+ds.1-7_source.changes dpkg-buildpackage: info: source package dask.distributed dpkg-buildpackage: info: source version 2024.5.2+ds.1-7 dpkg-buildpackage: info: source distribution unstable @@ -1472,196 +1508,196 @@ creating build creating build/lib creating build/lib/distributed -copying distributed/security.py -> build/lib/distributed -copying distributed/pytest_resourceleaks.py -> build/lib/distributed -copying distributed/pubsub.py -> build/lib/distributed -copying distributed/config.py -> build/lib/distributed -copying distributed/cfexecutor.py -> build/lib/distributed -copying distributed/_asyncio.py -> build/lib/distributed -copying distributed/nanny.py -> build/lib/distributed -copying distributed/client.py -> build/lib/distributed -copying distributed/recreate_tasks.py -> build/lib/distributed -copying distributed/system.py -> build/lib/distributed -copying distributed/profile.py -> build/lib/distributed -copying distributed/cluster_dump.py -> build/lib/distributed -copying distributed/spans.py -> build/lib/distributed -copying distributed/versions.py -> build/lib/distributed copying distributed/node.py -> build/lib/distributed -copying distributed/proctitle.py -> build/lib/distributed -copying distributed/utils.py -> build/lib/distributed -copying distributed/chaos.py -> build/lib/distributed -copying distributed/worker_client.py -> build/lib/distributed +copying distributed/nanny.py -> build/lib/distributed +copying distributed/metrics.py -> build/lib/distributed copying distributed/sizeof.py -> build/lib/distributed -copying distributed/_signals.py -> build/lib/distributed +copying distributed/semaphore.py -> build/lib/distributed +copying distributed/utils_comm.py -> build/lib/distributed copying distributed/_stories.py -> build/lib/distributed -copying distributed/publish.py -> build/lib/distributed -copying distributed/scheduler.py -> build/lib/distributed -copying distributed/itertools.py -> build/lib/distributed -copying distributed/core.py -> build/lib/distributed -copying distributed/batched.py -> build/lib/distributed +copying distributed/worker_state_machine.py -> build/lib/distributed +copying distributed/worker_client.py -> build/lib/distributed +copying distributed/spill.py -> build/lib/distributed copying distributed/event.py -> build/lib/distributed -copying distributed/process.py -> build/lib/distributed -copying distributed/worker.py -> build/lib/distributed copying distributed/lock.py -> build/lib/distributed -copying distributed/actor.py -> build/lib/distributed -copying distributed/preloading.py -> build/lib/distributed -copying distributed/_version.py -> build/lib/distributed -copying distributed/active_memory_manager.py -> build/lib/distributed -copying distributed/multi_lock.py -> build/lib/distributed -copying distributed/spill.py -> build/lib/distributed -copying distributed/bokeh.py -> build/lib/distributed copying distributed/threadpoolexecutor.py -> build/lib/distributed +copying distributed/pubsub.py -> build/lib/distributed +copying distributed/chaos.py -> build/lib/distributed +copying distributed/client.py -> build/lib/distributed +copying distributed/worker_memory.py -> build/lib/distributed +copying distributed/counter.py -> build/lib/distributed copying distributed/stealing.py -> build/lib/distributed +copying distributed/worker.py -> build/lib/distributed +copying distributed/pytest_resourceleaks.py -> build/lib/distributed +copying distributed/utils_test.py -> build/lib/distributed +copying distributed/scheduler.py -> build/lib/distributed +copying distributed/multi_lock.py -> build/lib/distributed +copying distributed/core.py -> build/lib/distributed +copying distributed/bokeh.py -> build/lib/distributed +copying distributed/preloading.py -> build/lib/distributed +copying distributed/publish.py -> build/lib/distributed +copying distributed/exceptions.py -> build/lib/distributed copying distributed/__init__.py -> build/lib/distributed -copying distributed/queues.py -> build/lib/distributed -copying distributed/collections.py -> build/lib/distributed +copying distributed/profile.py -> build/lib/distributed +copying distributed/actor.py -> build/lib/distributed copying distributed/diskutils.py -> build/lib/distributed +copying distributed/process.py -> build/lib/distributed +copying distributed/variable.py -> build/lib/distributed copying distributed/compatibility.py -> build/lib/distributed -copying distributed/worker_state_machine.py -> build/lib/distributed -copying distributed/metrics.py -> build/lib/distributed -copying distributed/exceptions.py -> build/lib/distributed -copying distributed/counter.py -> build/lib/distributed +copying distributed/utils_perf.py -> build/lib/distributed +copying distributed/config.py -> build/lib/distributed copying distributed/system_monitor.py -> build/lib/distributed -copying distributed/worker_memory.py -> build/lib/distributed -copying distributed/semaphore.py -> build/lib/distributed +copying distributed/batched.py -> build/lib/distributed +copying distributed/security.py -> build/lib/distributed +copying distributed/cfexecutor.py -> build/lib/distributed copying distributed/objects.py -> build/lib/distributed +copying distributed/itertools.py -> build/lib/distributed +copying distributed/proctitle.py -> build/lib/distributed +copying distributed/system.py -> build/lib/distributed +copying distributed/cluster_dump.py -> build/lib/distributed copying distributed/_concurrent_futures_thread.py -> build/lib/distributed -copying distributed/utils_perf.py -> build/lib/distributed -copying distributed/utils_test.py -> build/lib/distributed -copying distributed/variable.py -> build/lib/distributed -copying distributed/utils_comm.py -> build/lib/distributed +copying distributed/_signals.py -> build/lib/distributed +copying distributed/active_memory_manager.py -> build/lib/distributed +copying distributed/spans.py -> build/lib/distributed +copying distributed/utils.py -> build/lib/distributed +copying distributed/_asyncio.py -> build/lib/distributed +copying distributed/recreate_tasks.py -> build/lib/distributed +copying distributed/_version.py -> build/lib/distributed +copying distributed/versions.py -> build/lib/distributed +copying distributed/collections.py -> build/lib/distributed +copying distributed/queues.py -> build/lib/distributed +creating build/lib/distributed/shuffle +copying distributed/shuffle/_pickle.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_arrow.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_merge.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_disk.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_shuffle.py -> build/lib/distributed/shuffle +copying distributed/shuffle/__init__.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_memory.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_core.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_rechunk.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_worker_plugin.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_comms.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_buffer.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_exceptions.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_limiter.py -> build/lib/distributed/shuffle +copying distributed/shuffle/_scheduler_plugin.py -> build/lib/distributed/shuffle +creating build/lib/distributed/http +copying distributed/http/proxy.py -> build/lib/distributed/http +copying distributed/http/prometheus.py -> build/lib/distributed/http +copying distributed/http/__init__.py -> build/lib/distributed/http +copying distributed/http/statics.py -> build/lib/distributed/http +copying distributed/http/health.py -> build/lib/distributed/http +copying distributed/http/utils.py -> build/lib/distributed/http +copying distributed/http/routing.py -> build/lib/distributed/http creating build/lib/distributed/deploy +copying distributed/deploy/local.py -> build/lib/distributed/deploy +copying distributed/deploy/adaptive.py -> build/lib/distributed/deploy +copying distributed/deploy/subprocess.py -> build/lib/distributed/deploy copying distributed/deploy/old_ssh.py -> build/lib/distributed/deploy +copying distributed/deploy/__init__.py -> build/lib/distributed/deploy copying distributed/deploy/adaptive_core.py -> build/lib/distributed/deploy +copying distributed/deploy/ssh.py -> build/lib/distributed/deploy copying distributed/deploy/utils.py -> build/lib/distributed/deploy -copying distributed/deploy/local.py -> build/lib/distributed/deploy copying distributed/deploy/cluster.py -> build/lib/distributed/deploy -copying distributed/deploy/__init__.py -> build/lib/distributed/deploy -copying distributed/deploy/adaptive.py -> build/lib/distributed/deploy copying distributed/deploy/spec.py -> build/lib/distributed/deploy -copying distributed/deploy/subprocess.py -> build/lib/distributed/deploy -copying distributed/deploy/ssh.py -> build/lib/distributed/deploy +creating build/lib/distributed/diagnostics +copying distributed/diagnostics/nvml.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/rmm.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/progressbar.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/task_stream.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/__init__.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/websocket.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/memray.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/plugin.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/graph_layout.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/cluster_dump.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/eventstream.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/progress.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/cudf.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/memory_sampler.py -> build/lib/distributed/diagnostics +copying distributed/diagnostics/progress_stream.py -> build/lib/distributed/diagnostics +creating build/lib/distributed/widgets +copying distributed/widgets/__init__.py -> build/lib/distributed/widgets creating build/lib/distributed/protocol -copying distributed/protocol/netcdf4.py -> build/lib/distributed/protocol -copying distributed/protocol/rmm.py -> build/lib/distributed/protocol -copying distributed/protocol/pickle.py -> build/lib/distributed/protocol copying distributed/protocol/sparse.py -> build/lib/distributed/protocol -copying distributed/protocol/numba.py -> build/lib/distributed/protocol -copying distributed/protocol/utils.py -> build/lib/distributed/protocol -copying distributed/protocol/serialize.py -> build/lib/distributed/protocol -copying distributed/protocol/compression.py -> build/lib/distributed/protocol +copying distributed/protocol/cuda.py -> build/lib/distributed/protocol copying distributed/protocol/h5py.py -> build/lib/distributed/protocol -copying distributed/protocol/torch.py -> build/lib/distributed/protocol -copying distributed/protocol/arrow.py -> build/lib/distributed/protocol +copying distributed/protocol/keras.py -> build/lib/distributed/protocol +copying distributed/protocol/serialize.py -> build/lib/distributed/protocol +copying distributed/protocol/rmm.py -> build/lib/distributed/protocol +copying distributed/protocol/utils_test.py -> build/lib/distributed/protocol copying distributed/protocol/core.py -> build/lib/distributed/protocol -copying distributed/protocol/cuda.py -> build/lib/distributed/protocol -copying distributed/protocol/scipy.py -> build/lib/distributed/protocol -copying distributed/protocol/cupy.py -> build/lib/distributed/protocol +copying distributed/protocol/torch.py -> build/lib/distributed/protocol +copying distributed/protocol/netcdf4.py -> build/lib/distributed/protocol copying distributed/protocol/__init__.py -> build/lib/distributed/protocol -copying distributed/protocol/keras.py -> build/lib/distributed/protocol +copying distributed/protocol/compression.py -> build/lib/distributed/protocol +copying distributed/protocol/scipy.py -> build/lib/distributed/protocol +copying distributed/protocol/arrow.py -> build/lib/distributed/protocol copying distributed/protocol/numpy.py -> build/lib/distributed/protocol -copying distributed/protocol/utils_test.py -> build/lib/distributed/protocol +copying distributed/protocol/utils.py -> build/lib/distributed/protocol +copying distributed/protocol/pickle.py -> build/lib/distributed/protocol +copying distributed/protocol/cupy.py -> build/lib/distributed/protocol +copying distributed/protocol/numba.py -> build/lib/distributed/protocol +creating build/lib/distributed/dashboard +copying distributed/dashboard/worker.py -> build/lib/distributed/dashboard +copying distributed/dashboard/scheduler.py -> build/lib/distributed/dashboard +copying distributed/dashboard/core.py -> build/lib/distributed/dashboard +copying distributed/dashboard/__init__.py -> build/lib/distributed/dashboard +copying distributed/dashboard/export_tool.py -> build/lib/distributed/dashboard +copying distributed/dashboard/utils.py -> build/lib/distributed/dashboard creating build/lib/distributed/comm -copying distributed/comm/inproc.py -> build/lib/distributed/comm -copying distributed/comm/utils.py -> build/lib/distributed/comm -copying distributed/comm/tcp.py -> build/lib/distributed/comm +copying distributed/comm/registry.py -> build/lib/distributed/comm copying distributed/comm/core.py -> build/lib/distributed/comm -copying distributed/comm/__init__.py -> build/lib/distributed/comm copying distributed/comm/addressing.py -> build/lib/distributed/comm -copying distributed/comm/registry.py -> build/lib/distributed/comm +copying distributed/comm/__init__.py -> build/lib/distributed/comm +copying distributed/comm/inproc.py -> build/lib/distributed/comm copying distributed/comm/ucx.py -> build/lib/distributed/comm +copying distributed/comm/tcp.py -> build/lib/distributed/comm +copying distributed/comm/utils.py -> build/lib/distributed/comm copying distributed/comm/ws.py -> build/lib/distributed/comm -creating build/lib/distributed/widgets -copying distributed/widgets/__init__.py -> build/lib/distributed/widgets creating build/lib/distributed/cli +copying distributed/cli/dask_worker.py -> build/lib/distributed/cli copying distributed/cli/dask_ssh.py -> build/lib/distributed/cli -copying distributed/cli/utils.py -> build/lib/distributed/cli -copying distributed/cli/dask_spec.py -> build/lib/distributed/cli copying distributed/cli/dask_scheduler.py -> build/lib/distributed/cli copying distributed/cli/__init__.py -> build/lib/distributed/cli -copying distributed/cli/dask_worker.py -> build/lib/distributed/cli -creating build/lib/distributed/diagnostics -copying distributed/diagnostics/rmm.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/cluster_dump.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/websocket.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/nvml.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/memory_sampler.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/progress_stream.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/memray.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/cudf.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/plugin.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/__init__.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/eventstream.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/task_stream.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/graph_layout.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/progressbar.py -> build/lib/distributed/diagnostics -copying distributed/diagnostics/progress.py -> build/lib/distributed/diagnostics -creating build/lib/distributed/shuffle -copying distributed/shuffle/_arrow.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_exceptions.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_rechunk.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_limiter.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_merge.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_pickle.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_worker_plugin.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_shuffle.py -> build/lib/distributed/shuffle -copying distributed/shuffle/__init__.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_buffer.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_scheduler_plugin.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_memory.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_comms.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_core.py -> build/lib/distributed/shuffle -copying distributed/shuffle/_disk.py -> build/lib/distributed/shuffle -creating build/lib/distributed/http -copying distributed/http/prometheus.py -> build/lib/distributed/http -copying distributed/http/utils.py -> build/lib/distributed/http -copying distributed/http/proxy.py -> build/lib/distributed/http -copying distributed/http/statics.py -> build/lib/distributed/http -copying distributed/http/__init__.py -> build/lib/distributed/http -copying distributed/http/routing.py -> build/lib/distributed/http -copying distributed/http/health.py -> build/lib/distributed/http -creating build/lib/distributed/dashboard -copying distributed/dashboard/utils.py -> build/lib/distributed/dashboard -copying distributed/dashboard/export_tool.py -> build/lib/distributed/dashboard -copying distributed/dashboard/scheduler.py -> build/lib/distributed/dashboard -copying distributed/dashboard/core.py -> build/lib/distributed/dashboard -copying distributed/dashboard/worker.py -> build/lib/distributed/dashboard -copying distributed/dashboard/__init__.py -> build/lib/distributed/dashboard -creating build/lib/distributed/widgets/templates -copying distributed/widgets/templates/__init__.py -> build/lib/distributed/widgets/templates -creating build/lib/distributed/http/static -copying distributed/http/static/__init__.py -> build/lib/distributed/http/static -creating build/lib/distributed/http/worker -copying distributed/http/worker/__init__.py -> build/lib/distributed/http/worker -creating build/lib/distributed/http/templates -copying distributed/http/templates/__init__.py -> build/lib/distributed/http/templates +copying distributed/cli/utils.py -> build/lib/distributed/cli +copying distributed/cli/dask_spec.py -> build/lib/distributed/cli creating build/lib/distributed/http/scheduler +copying distributed/http/scheduler/json.py -> build/lib/distributed/http/scheduler +copying distributed/http/scheduler/__init__.py -> build/lib/distributed/http/scheduler copying distributed/http/scheduler/api.py -> build/lib/distributed/http/scheduler copying distributed/http/scheduler/info.py -> build/lib/distributed/http/scheduler -copying distributed/http/scheduler/json.py -> build/lib/distributed/http/scheduler copying distributed/http/scheduler/missing_bokeh.py -> build/lib/distributed/http/scheduler -copying distributed/http/scheduler/__init__.py -> build/lib/distributed/http/scheduler +creating build/lib/distributed/http/static +copying distributed/http/static/__init__.py -> build/lib/distributed/http/static +creating build/lib/distributed/http/templates +copying distributed/http/templates/__init__.py -> build/lib/distributed/http/templates +creating build/lib/distributed/http/worker +copying distributed/http/worker/__init__.py -> build/lib/distributed/http/worker +creating build/lib/distributed/http/scheduler/prometheus +copying distributed/http/scheduler/prometheus/semaphore.py -> build/lib/distributed/http/scheduler/prometheus +copying distributed/http/scheduler/prometheus/stealing.py -> build/lib/distributed/http/scheduler/prometheus +copying distributed/http/scheduler/prometheus/core.py -> build/lib/distributed/http/scheduler/prometheus +copying distributed/http/scheduler/prometheus/__init__.py -> build/lib/distributed/http/scheduler/prometheus +creating build/lib/distributed/http/static/images +copying distributed/http/static/images/__init__.py -> build/lib/distributed/http/static/images creating build/lib/distributed/http/static/css copying distributed/http/static/css/__init__.py -> build/lib/distributed/http/static/css creating build/lib/distributed/http/static/js copying distributed/http/static/js/__init__.py -> build/lib/distributed/http/static/js -creating build/lib/distributed/http/static/images -copying distributed/http/static/images/__init__.py -> build/lib/distributed/http/static/images creating build/lib/distributed/http/worker/prometheus copying distributed/http/worker/prometheus/core.py -> build/lib/distributed/http/worker/prometheus copying distributed/http/worker/prometheus/__init__.py -> build/lib/distributed/http/worker/prometheus -creating build/lib/distributed/http/scheduler/prometheus -copying distributed/http/scheduler/prometheus/core.py -> build/lib/distributed/http/scheduler/prometheus -copying distributed/http/scheduler/prometheus/stealing.py -> build/lib/distributed/http/scheduler/prometheus -copying distributed/http/scheduler/prometheus/__init__.py -> build/lib/distributed/http/scheduler/prometheus -copying distributed/http/scheduler/prometheus/semaphore.py -> build/lib/distributed/http/scheduler/prometheus +creating build/lib/distributed/widgets/templates +copying distributed/widgets/templates/__init__.py -> build/lib/distributed/widgets/templates creating build/lib/distributed/dashboard/components -copying distributed/dashboard/components/rmm.py -> build/lib/distributed/dashboard/components copying distributed/dashboard/components/nvml.py -> build/lib/distributed/dashboard/components -copying distributed/dashboard/components/shared.py -> build/lib/distributed/dashboard/components -copying distributed/dashboard/components/scheduler.py -> build/lib/distributed/dashboard/components +copying distributed/dashboard/components/rmm.py -> build/lib/distributed/dashboard/components copying distributed/dashboard/components/worker.py -> build/lib/distributed/dashboard/components +copying distributed/dashboard/components/scheduler.py -> build/lib/distributed/dashboard/components copying distributed/dashboard/components/__init__.py -> build/lib/distributed/dashboard/components +copying distributed/dashboard/components/shared.py -> build/lib/distributed/dashboard/components creating build/lib/distributed/dashboard/templates copying distributed/dashboard/templates/__init__.py -> build/lib/distributed/dashboard/templates running egg_info @@ -2208,19 +2244,41 @@ copying distributed/tests/tls-key.pem -> build/lib/distributed/tests copying distributed/tests/tls-self-signed-cert.pem -> build/lib/distributed/tests copying distributed/tests/tls-self-signed-key.pem -> build/lib/distributed/tests -copying distributed/http/templates/base.html -> build/lib/distributed/http/templates +copying distributed/http/templates/call-stack.html -> build/lib/distributed/http/templates copying distributed/http/templates/status.html -> build/lib/distributed/http/templates -copying distributed/http/templates/worker-table.html -> build/lib/distributed/http/templates +copying distributed/http/templates/json-index.html -> build/lib/distributed/http/templates +copying distributed/http/templates/task.html -> build/lib/distributed/http/templates copying distributed/http/templates/simple.html -> build/lib/distributed/http/templates +copying distributed/http/templates/base.html -> build/lib/distributed/http/templates copying distributed/http/templates/main.html -> build/lib/distributed/http/templates +copying distributed/http/templates/worker-table.html -> build/lib/distributed/http/templates +copying distributed/http/templates/gpu.html -> build/lib/distributed/http/templates +copying distributed/http/templates/logs.html -> build/lib/distributed/http/templates copying distributed/http/templates/workers.html -> build/lib/distributed/http/templates -copying distributed/http/templates/task.html -> build/lib/distributed/http/templates copying distributed/http/templates/exceptions.html -> build/lib/distributed/http/templates -copying distributed/http/templates/gpu.html -> build/lib/distributed/http/templates -copying distributed/http/templates/json-index.html -> build/lib/distributed/http/templates copying distributed/http/templates/worker.html -> build/lib/distributed/http/templates -copying distributed/http/templates/logs.html -> build/lib/distributed/http/templates -copying distributed/http/templates/call-stack.html -> build/lib/distributed/http/templates +creating build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/__init__.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_buffer.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_comm_buffer.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_core.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_disk_buffer.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_graph.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_limiter.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_memory_buffer.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_merge.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_merge_column_and_index.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_metrics.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_pickle.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_read_write_lock.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_rechunk.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_shuffle.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/test_shuffle_plugins.py -> build/lib/distributed/shuffle/tests +copying distributed/shuffle/tests/utils.py -> build/lib/distributed/shuffle/tests +creating build/lib/distributed/http/tests +copying distributed/http/tests/__init__.py -> build/lib/distributed/http/tests +copying distributed/http/tests/test_core.py -> build/lib/distributed/http/tests +copying distributed/http/tests/test_routing.py -> build/lib/distributed/http/tests creating build/lib/distributed/deploy/tests copying distributed/deploy/tests/test_adaptive.py -> build/lib/distributed/deploy/tests copying distributed/deploy/tests/test_adaptive_core.py -> build/lib/distributed/deploy/tests @@ -2232,6 +2290,24 @@ copying distributed/deploy/tests/test_spec_cluster.py -> build/lib/distributed/deploy/tests copying distributed/deploy/tests/test_ssh.py -> build/lib/distributed/deploy/tests copying distributed/deploy/tests/test_subprocess.py -> build/lib/distributed/deploy/tests +creating build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_cluster_dump_plugin.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_cudf_diagnostics.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_eventstream.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_graph_layout.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_install_plugin.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_memory_sampler.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_memray.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_nanny_plugin.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_nvml.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_progress.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_progress_stream.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_progress_widgets.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_progressbar.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_rmm_diagnostics.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_scheduler_plugin.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_task_stream.py -> build/lib/distributed/diagnostics/tests +copying distributed/diagnostics/tests/test_worker_plugin.py -> build/lib/distributed/diagnostics/tests creating build/lib/distributed/protocol/tests copying distributed/protocol/tests/__init__.py -> build/lib/distributed/protocol/tests copying distributed/protocol/tests/test_arrow.py -> build/lib/distributed/protocol/tests @@ -2257,6 +2333,12 @@ copying distributed/protocol/tests/test_torch.py -> build/lib/distributed/protocol/tests copying distributed/protocol/tests/test_utils.py -> build/lib/distributed/protocol/tests copying distributed/protocol/tests/test_utils_test.py -> build/lib/distributed/protocol/tests +copying distributed/dashboard/theme.yaml -> build/lib/distributed/dashboard +creating build/lib/distributed/dashboard/tests +copying distributed/dashboard/tests/test_bokeh.py -> build/lib/distributed/dashboard/tests +copying distributed/dashboard/tests/test_components.py -> build/lib/distributed/dashboard/tests +copying distributed/dashboard/tests/test_scheduler_bokeh.py -> build/lib/distributed/dashboard/tests +copying distributed/dashboard/tests/test_worker_bokeh.py -> build/lib/distributed/dashboard/tests creating build/lib/distributed/comm/tests copying distributed/comm/tests/test_comms.py -> build/lib/distributed/comm/tests copying distributed/comm/tests/test_tcp.py -> build/lib/distributed/comm/tests @@ -2270,52 +2352,30 @@ copying distributed/cli/tests/test_dask_ssh.py -> build/lib/distributed/cli/tests copying distributed/cli/tests/test_dask_worker.py -> build/lib/distributed/cli/tests copying distributed/cli/tests/test_tls_cli.py -> build/lib/distributed/cli/tests -creating build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_cluster_dump_plugin.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_cudf_diagnostics.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_eventstream.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_graph_layout.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_install_plugin.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_memory_sampler.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_memray.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_nanny_plugin.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_nvml.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_progress.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_progress_stream.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_progress_widgets.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_progressbar.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_rmm_diagnostics.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_scheduler_plugin.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_task_stream.py -> build/lib/distributed/diagnostics/tests -copying distributed/diagnostics/tests/test_worker_plugin.py -> build/lib/distributed/diagnostics/tests -creating build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/__init__.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_buffer.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_comm_buffer.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_core.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_disk_buffer.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_graph.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_limiter.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_memory_buffer.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_merge.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_merge_column_and_index.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_metrics.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_pickle.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_read_write_lock.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_rechunk.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_shuffle.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/test_shuffle_plugins.py -> build/lib/distributed/shuffle/tests -copying distributed/shuffle/tests/utils.py -> build/lib/distributed/shuffle/tests -creating build/lib/distributed/http/tests -copying distributed/http/tests/__init__.py -> build/lib/distributed/http/tests -copying distributed/http/tests/test_core.py -> build/lib/distributed/http/tests -copying distributed/http/tests/test_routing.py -> build/lib/distributed/http/tests -copying distributed/dashboard/theme.yaml -> build/lib/distributed/dashboard -creating build/lib/distributed/dashboard/tests -copying distributed/dashboard/tests/test_bokeh.py -> build/lib/distributed/dashboard/tests -copying distributed/dashboard/tests/test_components.py -> build/lib/distributed/dashboard/tests -copying distributed/dashboard/tests/test_scheduler_bokeh.py -> build/lib/distributed/dashboard/tests -copying distributed/dashboard/tests/test_worker_bokeh.py -> build/lib/distributed/dashboard/tests +creating build/lib/distributed/http/scheduler/tests +copying distributed/http/scheduler/tests/test_missing_bokeh.py -> build/lib/distributed/http/scheduler/tests +copying distributed/http/scheduler/tests/test_scheduler_http.py -> build/lib/distributed/http/scheduler/tests +copying distributed/http/scheduler/tests/test_semaphore_http.py -> build/lib/distributed/http/scheduler/tests +copying distributed/http/scheduler/tests/test_stealing_http.py -> build/lib/distributed/http/scheduler/tests +copying distributed/http/static/individual-cluster-map.html -> build/lib/distributed/http/static +creating build/lib/distributed/http/worker/tests +copying distributed/http/worker/tests/test_worker_http.py -> build/lib/distributed/http/worker/tests +copying distributed/http/static/images/dask-logo.svg -> build/lib/distributed/http/static/images +copying distributed/http/static/images/fa-bars.svg -> build/lib/distributed/http/static/images +copying distributed/http/static/images/favicon.ico -> build/lib/distributed/http/static/images +copying distributed/http/static/images/jupyter.svg -> build/lib/distributed/http/static/images +copying distributed/http/static/images/numpy.png -> build/lib/distributed/http/static/images +copying distributed/http/static/images/pandas.png -> build/lib/distributed/http/static/images +copying distributed/http/static/images/python.png -> build/lib/distributed/http/static/images +copying distributed/http/static/css/base.css -> build/lib/distributed/http/static/css +copying distributed/http/static/css/gpu.css -> build/lib/distributed/http/static/css +copying distributed/http/static/css/individual-cluster-map.css -> build/lib/distributed/http/static/css +copying distributed/http/static/css/sortable.min.css -> build/lib/distributed/http/static/css +copying distributed/http/static/css/status.css -> build/lib/distributed/http/static/css +copying distributed/http/static/js/anime.min.js -> build/lib/distributed/http/static/js +copying distributed/http/static/js/individual-cluster-map.js -> build/lib/distributed/http/static/js +copying distributed/http/static/js/reconnecting-websocket.min.js -> build/lib/distributed/http/static/js +copying distributed/http/static/js/sortable.min.js -> build/lib/distributed/http/static/js copying distributed/widgets/templates/client.html.j2 -> build/lib/distributed/widgets/templates copying distributed/widgets/templates/cluster.html.j2 -> build/lib/distributed/widgets/templates copying distributed/widgets/templates/computation.html.j2 -> build/lib/distributed/widgets/templates @@ -2331,30 +2391,6 @@ copying distributed/widgets/templates/task_state.html.j2 -> build/lib/distributed/widgets/templates copying distributed/widgets/templates/who_has.html.j2 -> build/lib/distributed/widgets/templates copying distributed/widgets/templates/worker_state.html.j2 -> build/lib/distributed/widgets/templates -copying distributed/http/static/individual-cluster-map.html -> build/lib/distributed/http/static -creating build/lib/distributed/http/worker/tests -copying distributed/http/worker/tests/test_worker_http.py -> build/lib/distributed/http/worker/tests -creating build/lib/distributed/http/scheduler/tests -copying distributed/http/scheduler/tests/test_missing_bokeh.py -> build/lib/distributed/http/scheduler/tests -copying distributed/http/scheduler/tests/test_scheduler_http.py -> build/lib/distributed/http/scheduler/tests -copying distributed/http/scheduler/tests/test_semaphore_http.py -> build/lib/distributed/http/scheduler/tests -copying distributed/http/scheduler/tests/test_stealing_http.py -> build/lib/distributed/http/scheduler/tests -copying distributed/http/static/css/base.css -> build/lib/distributed/http/static/css -copying distributed/http/static/css/gpu.css -> build/lib/distributed/http/static/css -copying distributed/http/static/css/individual-cluster-map.css -> build/lib/distributed/http/static/css -copying distributed/http/static/css/sortable.min.css -> build/lib/distributed/http/static/css -copying distributed/http/static/css/status.css -> build/lib/distributed/http/static/css -copying distributed/http/static/js/anime.min.js -> build/lib/distributed/http/static/js -copying distributed/http/static/js/individual-cluster-map.js -> build/lib/distributed/http/static/js -copying distributed/http/static/js/reconnecting-websocket.min.js -> build/lib/distributed/http/static/js -copying distributed/http/static/js/sortable.min.js -> build/lib/distributed/http/static/js -copying distributed/http/static/images/dask-logo.svg -> build/lib/distributed/http/static/images -copying distributed/http/static/images/fa-bars.svg -> build/lib/distributed/http/static/images -copying distributed/http/static/images/favicon.ico -> build/lib/distributed/http/static/images -copying distributed/http/static/images/jupyter.svg -> build/lib/distributed/http/static/images -copying distributed/http/static/images/numpy.png -> build/lib/distributed/http/static/images -copying distributed/http/static/images/pandas.png -> build/lib/distributed/http/static/images -copying distributed/http/static/images/python.png -> build/lib/distributed/http/static/images copying distributed/dashboard/templates/performance_report.html -> build/lib/distributed/dashboard/templates UPDATING build/lib/distributed/_version.py set build/lib/distributed/_version.py to '2024.5.2' @@ -2364,429 +2400,429 @@ creating build/bdist.linux-x86_64 creating build/bdist.linux-x86_64/wheel creating build/bdist.linux-x86_64/wheel/distributed -copying build/lib/distributed/security.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/pytest_resourceleaks.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/pubsub.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/config.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/cfexecutor.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/distributed.yaml -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/_asyncio.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/nanny.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/client.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/recreate_tasks.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/system.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/profile.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/cluster_dump.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/spans.py -> build/bdist.linux-x86_64/wheel/./distributed -creating build/bdist.linux-x86_64/wheel/distributed/tests -copying build/lib/distributed/tests/test_active_memory_manager.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-self-signed-key.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_publish.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_worker_client.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_reschedule.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-ca-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_worker_metrics.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_spill.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_queues.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_parse_stdout.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_failed_workers.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_imports.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_semaphore.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_locks.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_stories.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_counter.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_threadpoolexecutor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_asyncio.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_init.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_priorities.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_utils_comm.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_metrics.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_core.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_spec.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_security.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_variable.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_client.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_computations.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/make_tls_certs.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_jupyter.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_tls_functional.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_diskutils.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_as_completed.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_asyncprocess.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_system.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_utils_perf.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_chaos.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_sizeof.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_batched.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_nanny.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_compatibility.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_client_loop.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-self-signed-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_spans.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-key.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_cluster_dump.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_worker_memory.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_versions.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-key-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_collections.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_profile.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_resources.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_worker.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_client_executor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_steal.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_config.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_actor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_worker_state_machine.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-cert-chain.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_preload.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_cancelled_state.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_events.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_pubsub.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_dask_collections.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/tls-ca-key.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_multi_locks.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_deadline.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_stress.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_itertools.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/tests/test_system_monitor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests -copying build/lib/distributed/versions.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/node.py -> build/bdist.linux-x86_64/wheel/./distributed -creating build/bdist.linux-x86_64/wheel/distributed/deploy -copying build/lib/distributed/deploy/old_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/adaptive_core.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -creating build/bdist.linux-x86_64/wheel/distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_adaptive_core.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_spec_cluster.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_subprocess.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_deploy_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_local.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_adaptive.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_old_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_cluster.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_slow_adaptive.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/tests/test_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests -copying build/lib/distributed/deploy/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/local.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/cluster.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/adaptive.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/spec.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/subprocess.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/deploy/ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy -copying build/lib/distributed/proctitle.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/utils.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/chaos.py -> build/bdist.linux-x86_64/wheel/./distributed -creating build/bdist.linux-x86_64/wheel/distributed/protocol -copying build/lib/distributed/protocol/netcdf4.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/sparse.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/numba.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -creating build/bdist.linux-x86_64/wheel/distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_compression.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_numpy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_arrow.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_protocol_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_keras.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_scipy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_cupy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_sparse.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_h5py.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_numba.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_serialize.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_protocol.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_pandas.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_highlevelgraph.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_torch.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_collection_cuda.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_to_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_collection.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/tests/test_netcdf4.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests -copying build/lib/distributed/protocol/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/serialize.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/compression.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/h5py.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/torch.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/arrow.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/core.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/cuda.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/scipy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/cupy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/keras.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/numpy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/protocol/utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol -copying build/lib/distributed/worker_client.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/sizeof.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/_signals.py -> build/bdist.linux-x86_64/wheel/./distributed -creating build/bdist.linux-x86_64/wheel/distributed/comm -copying build/lib/distributed/comm/inproc.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -creating build/bdist.linux-x86_64/wheel/distributed/comm/tests -copying build/lib/distributed/comm/tests/test_comms.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests -copying build/lib/distributed/comm/tests/test_tcp.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests -copying build/lib/distributed/comm/tests/test_ws.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests -copying build/lib/distributed/comm/tests/test_ucx.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests -copying build/lib/distributed/comm/tests/test_ucx_config.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests -copying build/lib/distributed/comm/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/tcp.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/core.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/addressing.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/registry.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/ucx.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -copying build/lib/distributed/comm/ws.py -> build/bdist.linux-x86_64/wheel/./distributed/comm -creating build/bdist.linux-x86_64/wheel/distributed/widgets -copying build/lib/distributed/widgets/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/widgets -creating build/bdist.linux-x86_64/wheel/distributed/widgets/templates -copying build/lib/distributed/widgets/templates/security.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/who_has.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/local_cluster.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/future.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/scheduler_info.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/computation.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/process_interface.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/log.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/logs.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/cluster.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/task_state.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/has_what.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/worker_state.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/client.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/widgets/templates/scheduler.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates -copying build/lib/distributed/_stories.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/publish.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/itertools.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/core.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/py.typed -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/batched.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/event.py -> build/bdist.linux-x86_64/wheel/./distributed -creating build/bdist.linux-x86_64/wheel/distributed/cli -creating build/bdist.linux-x86_64/wheel/distributed/cli/tests -copying build/lib/distributed/cli/tests/test_tls_cli.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests -copying build/lib/distributed/cli/tests/test_dask_worker.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests -copying build/lib/distributed/cli/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests -copying build/lib/distributed/cli/tests/test_dask_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests -copying build/lib/distributed/cli/tests/test_dask_spec.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests -copying build/lib/distributed/cli/tests/test_dask_scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests -copying build/lib/distributed/cli/dask_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/cli -copying build/lib/distributed/cli/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/cli -copying build/lib/distributed/cli/dask_spec.py -> build/bdist.linux-x86_64/wheel/./distributed/cli -copying build/lib/distributed/cli/dask_scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/cli -copying build/lib/distributed/cli/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/cli -copying build/lib/distributed/cli/dask_worker.py -> build/bdist.linux-x86_64/wheel/./distributed/cli -copying build/lib/distributed/process.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/worker.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/lock.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/actor.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/preloading.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/_version.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/active_memory_manager.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/multi_lock.py -> build/bdist.linux-x86_64/wheel/./distributed -creating build/bdist.linux-x86_64/wheel/distributed/diagnostics -copying build/lib/distributed/diagnostics/rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/cluster_dump.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -creating build/bdist.linux-x86_64/wheel/distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_cudf_diagnostics.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_memray.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_rmm_diagnostics.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_nanny_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_graph_layout.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_cluster_dump_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_scheduler_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_install_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_eventstream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_nvml.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_progressbar.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_memory_sampler.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_worker_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_progress_widgets.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_progress_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_task_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/tests/test_progress.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests -copying build/lib/distributed/diagnostics/websocket.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/nvml.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/memory_sampler.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/progress_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/memray.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/cudf.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/eventstream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/task_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/graph_layout.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/progressbar.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/diagnostics/progress.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics -copying build/lib/distributed/spill.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/threadpoolexecutor.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/stealing.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed creating build/bdist.linux-x86_64/wheel/distributed/shuffle +copying build/lib/distributed/shuffle/_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle copying build/lib/distributed/shuffle/_arrow.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_exceptions.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_merge.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_disk.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_shuffle.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_memory.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle creating build/bdist.linux-x86_64/wheel/distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/test_graph.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/test_metrics.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_core.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_comm_buffer.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_limiter.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/test_merge_column_and_index.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/test_disk_buffer.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_buffer.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/test_read_write_lock.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_shuffle.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/test_disk_buffer.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_memory_buffer.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/test_metrics.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/test_graph.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_shuffle_plugins.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/test_merge_column_and_index.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/test_rechunk.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests -copying build/lib/distributed/shuffle/tests/test_read_write_lock.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests copying build/lib/distributed/shuffle/tests/test_merge.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/tests/test_rechunk.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle/tests +copying build/lib/distributed/shuffle/_core.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle copying build/lib/distributed/shuffle/_rechunk.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_limiter.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_merge.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle copying build/lib/distributed/shuffle/_worker_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_shuffle.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_comms.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle copying build/lib/distributed/shuffle/_buffer.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_exceptions.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle +copying build/lib/distributed/shuffle/_limiter.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle copying build/lib/distributed/shuffle/_scheduler_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_memory.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_comms.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_core.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/shuffle/_disk.py -> build/bdist.linux-x86_64/wheel/./distributed/shuffle -copying build/lib/distributed/queues.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/collections.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/diskutils.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/compatibility.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/worker_state_machine.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/metrics.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/exceptions.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/node.py -> build/bdist.linux-x86_64/wheel/./distributed creating build/bdist.linux-x86_64/wheel/distributed/http +creating build/bdist.linux-x86_64/wheel/distributed/http/scheduler +copying build/lib/distributed/http/scheduler/json.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler +copying build/lib/distributed/http/scheduler/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler +creating build/bdist.linux-x86_64/wheel/distributed/http/scheduler/tests +copying build/lib/distributed/http/scheduler/tests/test_missing_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests +copying build/lib/distributed/http/scheduler/tests/test_stealing_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests +copying build/lib/distributed/http/scheduler/tests/test_scheduler_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests +copying build/lib/distributed/http/scheduler/tests/test_semaphore_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests +copying build/lib/distributed/http/scheduler/api.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler +copying build/lib/distributed/http/scheduler/info.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler +copying build/lib/distributed/http/scheduler/missing_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler +creating build/bdist.linux-x86_64/wheel/distributed/http/scheduler/prometheus +copying build/lib/distributed/http/scheduler/prometheus/semaphore.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus +copying build/lib/distributed/http/scheduler/prometheus/stealing.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus +copying build/lib/distributed/http/scheduler/prometheus/core.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus +copying build/lib/distributed/http/scheduler/prometheus/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus +copying build/lib/distributed/http/proxy.py -> build/bdist.linux-x86_64/wheel/./distributed/http +copying build/lib/distributed/http/prometheus.py -> build/bdist.linux-x86_64/wheel/./distributed/http +copying build/lib/distributed/http/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http +creating build/bdist.linux-x86_64/wheel/distributed/http/tests +copying build/lib/distributed/http/tests/test_core.py -> build/bdist.linux-x86_64/wheel/./distributed/http/tests +copying build/lib/distributed/http/tests/test_routing.py -> build/bdist.linux-x86_64/wheel/./distributed/http/tests +copying build/lib/distributed/http/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/tests +copying build/lib/distributed/http/statics.py -> build/bdist.linux-x86_64/wheel/./distributed/http +copying build/lib/distributed/http/health.py -> build/bdist.linux-x86_64/wheel/./distributed/http +copying build/lib/distributed/http/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/http creating build/bdist.linux-x86_64/wheel/distributed/http/static +creating build/bdist.linux-x86_64/wheel/distributed/http/static/images +copying build/lib/distributed/http/static/images/jupyter.svg -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/favicon.ico -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/pandas.png -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/numpy.png -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/fa-bars.svg -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/python.png -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images +copying build/lib/distributed/http/static/images/dask-logo.svg -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images creating build/bdist.linux-x86_64/wheel/distributed/http/static/css +copying build/lib/distributed/http/static/css/gpu.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css copying build/lib/distributed/http/static/css/base.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css copying build/lib/distributed/http/static/css/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css -copying build/lib/distributed/http/static/css/gpu.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css copying build/lib/distributed/http/static/css/status.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css -copying build/lib/distributed/http/static/css/individual-cluster-map.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css copying build/lib/distributed/http/static/css/sortable.min.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css +copying build/lib/distributed/http/static/css/individual-cluster-map.css -> build/bdist.linux-x86_64/wheel/./distributed/http/static/css +copying build/lib/distributed/http/static/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static +copying build/lib/distributed/http/static/individual-cluster-map.html -> build/bdist.linux-x86_64/wheel/./distributed/http/static creating build/bdist.linux-x86_64/wheel/distributed/http/static/js -copying build/lib/distributed/http/static/js/anime.min.js -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js copying build/lib/distributed/http/static/js/individual-cluster-map.js -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js -copying build/lib/distributed/http/static/js/reconnecting-websocket.min.js -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js -copying build/lib/distributed/http/static/js/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js copying build/lib/distributed/http/static/js/sortable.min.js -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js -copying build/lib/distributed/http/static/individual-cluster-map.html -> build/bdist.linux-x86_64/wheel/./distributed/http/static -copying build/lib/distributed/http/static/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static -creating build/bdist.linux-x86_64/wheel/distributed/http/static/images -copying build/lib/distributed/http/static/images/python.png -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/dask-logo.svg -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/fa-bars.svg -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/jupyter.svg -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/numpy.png -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/pandas.png -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -copying build/lib/distributed/http/static/images/favicon.ico -> build/bdist.linux-x86_64/wheel/./distributed/http/static/images -creating build/bdist.linux-x86_64/wheel/distributed/http/tests -copying build/lib/distributed/http/tests/test_routing.py -> build/bdist.linux-x86_64/wheel/./distributed/http/tests -copying build/lib/distributed/http/tests/test_core.py -> build/bdist.linux-x86_64/wheel/./distributed/http/tests -copying build/lib/distributed/http/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/tests -copying build/lib/distributed/http/prometheus.py -> build/bdist.linux-x86_64/wheel/./distributed/http -copying build/lib/distributed/http/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/http -copying build/lib/distributed/http/proxy.py -> build/bdist.linux-x86_64/wheel/./distributed/http -copying build/lib/distributed/http/statics.py -> build/bdist.linux-x86_64/wheel/./distributed/http -copying build/lib/distributed/http/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http -creating build/bdist.linux-x86_64/wheel/distributed/http/worker -creating build/bdist.linux-x86_64/wheel/distributed/http/worker/tests -copying build/lib/distributed/http/worker/tests/test_worker_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker/tests -copying build/lib/distributed/http/worker/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker -creating build/bdist.linux-x86_64/wheel/distributed/http/worker/prometheus -copying build/lib/distributed/http/worker/prometheus/core.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker/prometheus -copying build/lib/distributed/http/worker/prometheus/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker/prometheus +copying build/lib/distributed/http/static/js/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js +copying build/lib/distributed/http/static/js/reconnecting-websocket.min.js -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js +copying build/lib/distributed/http/static/js/anime.min.js -> build/bdist.linux-x86_64/wheel/./distributed/http/static/js creating build/bdist.linux-x86_64/wheel/distributed/http/templates -copying build/lib/distributed/http/templates/base.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/call-stack.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/templates/status.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/worker-table.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/json-index.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/task.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/templates/simple.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/base.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/templates/main.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/worker-table.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/gpu.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates +copying build/lib/distributed/http/templates/logs.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/templates/workers.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/task.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/templates/exceptions.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/gpu.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/json-index.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/templates/worker.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/logs.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates -copying build/lib/distributed/http/templates/call-stack.html -> build/bdist.linux-x86_64/wheel/./distributed/http/templates copying build/lib/distributed/http/routing.py -> build/bdist.linux-x86_64/wheel/./distributed/http -copying build/lib/distributed/http/health.py -> build/bdist.linux-x86_64/wheel/./distributed/http -creating build/bdist.linux-x86_64/wheel/distributed/http/scheduler -creating build/bdist.linux-x86_64/wheel/distributed/http/scheduler/tests -copying build/lib/distributed/http/scheduler/tests/test_semaphore_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests -copying build/lib/distributed/http/scheduler/tests/test_missing_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests -copying build/lib/distributed/http/scheduler/tests/test_stealing_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests -copying build/lib/distributed/http/scheduler/tests/test_scheduler_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/tests -copying build/lib/distributed/http/scheduler/api.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler -copying build/lib/distributed/http/scheduler/info.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler -copying build/lib/distributed/http/scheduler/json.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler -copying build/lib/distributed/http/scheduler/missing_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler -copying build/lib/distributed/http/scheduler/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler -creating build/bdist.linux-x86_64/wheel/distributed/http/scheduler/prometheus -copying build/lib/distributed/http/scheduler/prometheus/core.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus -copying build/lib/distributed/http/scheduler/prometheus/stealing.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus -copying build/lib/distributed/http/scheduler/prometheus/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus -copying build/lib/distributed/http/scheduler/prometheus/semaphore.py -> build/bdist.linux-x86_64/wheel/./distributed/http/scheduler/prometheus +creating build/bdist.linux-x86_64/wheel/distributed/http/worker +copying build/lib/distributed/http/worker/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker +creating build/bdist.linux-x86_64/wheel/distributed/http/worker/tests +copying build/lib/distributed/http/worker/tests/test_worker_http.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker/tests +creating build/bdist.linux-x86_64/wheel/distributed/http/worker/prometheus +copying build/lib/distributed/http/worker/prometheus/core.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker/prometheus +copying build/lib/distributed/http/worker/prometheus/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/http/worker/prometheus +copying build/lib/distributed/nanny.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/metrics.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/sizeof.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/semaphore.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/utils_comm.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/deploy +copying build/lib/distributed/deploy/local.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/adaptive.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/subprocess.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/old_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/adaptive_core.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +creating build/bdist.linux-x86_64/wheel/distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_slow_adaptive.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_adaptive_core.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_deploy_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_adaptive.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_spec_cluster.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_local.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_cluster.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_subprocess.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/tests/test_old_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy/tests +copying build/lib/distributed/deploy/ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/cluster.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/deploy/spec.py -> build/bdist.linux-x86_64/wheel/./distributed/deploy +copying build/lib/distributed/_stories.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/worker_state_machine.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/worker_client.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/spill.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/event.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/diagnostics +copying build/lib/distributed/diagnostics/nvml.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/progressbar.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/task_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/websocket.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/memray.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +creating build/bdist.linux-x86_64/wheel/distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_install_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_worker_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_graph_layout.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_task_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_nvml.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_cluster_dump_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_memory_sampler.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_nanny_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_progress.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_memray.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_scheduler_plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_progressbar.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_progress_widgets.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_progress_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_cudf_diagnostics.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_eventstream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/tests/test_rmm_diagnostics.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics/tests +copying build/lib/distributed/diagnostics/plugin.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/graph_layout.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/cluster_dump.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/eventstream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/progress.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/cudf.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/memory_sampler.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/diagnostics/progress_stream.py -> build/bdist.linux-x86_64/wheel/./distributed/diagnostics +copying build/lib/distributed/lock.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/threadpoolexecutor.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/pubsub.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/chaos.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/client.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/worker_memory.py -> build/bdist.linux-x86_64/wheel/./distributed copying build/lib/distributed/counter.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/stealing.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/widgets +copying build/lib/distributed/widgets/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/widgets +creating build/bdist.linux-x86_64/wheel/distributed/widgets/templates +copying build/lib/distributed/widgets/templates/log.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/who_has.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/local_cluster.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/security.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/client.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/computation.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/task_state.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/has_what.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/future.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/cluster.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/logs.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/scheduler.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/worker_state.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/scheduler_info.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/widgets/templates/process_interface.html.j2 -> build/bdist.linux-x86_64/wheel/./distributed/widgets/templates +copying build/lib/distributed/worker.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/protocol +copying build/lib/distributed/protocol/sparse.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/cuda.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/h5py.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/keras.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/serialize.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/core.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/torch.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/netcdf4.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/compression.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +creating build/bdist.linux-x86_64/wheel/distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_serialize.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_scipy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_sparse.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_protocol.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_collection.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_arrow.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_numba.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_highlevelgraph.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_cupy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_numpy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_pandas.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_netcdf4.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_compression.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_protocol_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_to_pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_keras.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_torch.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_collection_cuda.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/tests/test_h5py.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol/tests +copying build/lib/distributed/protocol/scipy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/arrow.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/numpy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/pickle.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/cupy.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/protocol/numba.py -> build/bdist.linux-x86_64/wheel/./distributed/protocol +copying build/lib/distributed/pytest_resourceleaks.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/multi_lock.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/core.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/preloading.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/publish.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/exceptions.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/profile.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/actor.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/diskutils.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/py.typed -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/process.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/tests +copying build/lib/distributed/tests/test_events.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_asyncio.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_counter.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-self-signed-key.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_system_monitor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_steal.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_threadpoolexecutor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_security.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_semaphore.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_jupyter.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_worker_metrics.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_stories.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_queues.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_failed_workers.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_worker_client.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_core.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_stress.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_nanny.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_cancelled_state.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_system.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_dask_collections.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_imports.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_profile.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_collections.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-self-signed-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_worker_state_machine.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_active_memory_manager.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_parse_stdout.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_batched.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-cert-chain.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_deadline.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_cluster_dump.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_diskutils.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_spans.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_pubsub.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_publish.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_sizeof.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_as_completed.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-ca-key.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_spill.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-key.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_client_loop.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_variable.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-ca-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_utils_perf.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_versions.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_computations.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_utils_comm.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_locks.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_chaos.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_worker_memory.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_spec.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_client_executor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-key-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_actor.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_reschedule.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_client.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_utils.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_init.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/tls-cert.pem -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_worker.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_itertools.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_tls_functional.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_asyncprocess.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_config.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_resources.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_metrics.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_preload.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_compatibility.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/make_tls_certs.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_priorities.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/tests/test_multi_locks.py -> build/bdist.linux-x86_64/wheel/./distributed/tests +copying build/lib/distributed/variable.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/compatibility.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/utils_perf.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/distributed.yaml -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/config.py -> build/bdist.linux-x86_64/wheel/./distributed copying build/lib/distributed/system_monitor.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/worker_memory.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/semaphore.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/batched.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/security.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/cfexecutor.py -> build/bdist.linux-x86_64/wheel/./distributed copying build/lib/distributed/objects.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/itertools.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/proctitle.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/system.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/cluster_dump.py -> build/bdist.linux-x86_64/wheel/./distributed copying build/lib/distributed/_concurrent_futures_thread.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/utils_perf.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/_signals.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/active_memory_manager.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/spans.py -> build/bdist.linux-x86_64/wheel/./distributed copying build/lib/distributed/distributed-schema.yaml -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/utils_test.py -> build/bdist.linux-x86_64/wheel/./distributed -copying build/lib/distributed/variable.py -> build/bdist.linux-x86_64/wheel/./distributed creating build/bdist.linux-x86_64/wheel/distributed/dashboard -creating build/bdist.linux-x86_64/wheel/distributed/dashboard/tests -copying build/lib/distributed/dashboard/tests/test_scheduler_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests -copying build/lib/distributed/dashboard/tests/test_components.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests -copying build/lib/distributed/dashboard/tests/test_worker_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests -copying build/lib/distributed/dashboard/tests/test_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests -copying build/lib/distributed/dashboard/theme.yaml -> build/bdist.linux-x86_64/wheel/./distributed/dashboard -copying build/lib/distributed/dashboard/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard -copying build/lib/distributed/dashboard/export_tool.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard -copying build/lib/distributed/dashboard/scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard -copying build/lib/distributed/dashboard/core.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard creating build/bdist.linux-x86_64/wheel/distributed/dashboard/components -copying build/lib/distributed/dashboard/components/rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components copying build/lib/distributed/dashboard/components/nvml.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components -copying build/lib/distributed/dashboard/components/shared.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components -copying build/lib/distributed/dashboard/components/scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components +copying build/lib/distributed/dashboard/components/rmm.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components copying build/lib/distributed/dashboard/components/worker.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components +copying build/lib/distributed/dashboard/components/scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components copying build/lib/distributed/dashboard/components/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components +copying build/lib/distributed/dashboard/components/shared.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/components copying build/lib/distributed/dashboard/worker.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard +copying build/lib/distributed/dashboard/scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard +copying build/lib/distributed/dashboard/core.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard +copying build/lib/distributed/dashboard/theme.yaml -> build/bdist.linux-x86_64/wheel/./distributed/dashboard copying build/lib/distributed/dashboard/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard +creating build/bdist.linux-x86_64/wheel/distributed/dashboard/tests +copying build/lib/distributed/dashboard/tests/test_worker_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests +copying build/lib/distributed/dashboard/tests/test_scheduler_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests +copying build/lib/distributed/dashboard/tests/test_bokeh.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests +copying build/lib/distributed/dashboard/tests/test_components.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/tests +copying build/lib/distributed/dashboard/export_tool.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard +copying build/lib/distributed/dashboard/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard creating build/bdist.linux-x86_64/wheel/distributed/dashboard/templates copying build/lib/distributed/dashboard/templates/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/templates copying build/lib/distributed/dashboard/templates/performance_report.html -> build/bdist.linux-x86_64/wheel/./distributed/dashboard/templates -copying build/lib/distributed/utils_comm.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/utils.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/comm +copying build/lib/distributed/comm/registry.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/core.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/addressing.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/inproc.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +creating build/bdist.linux-x86_64/wheel/distributed/comm/tests +copying build/lib/distributed/comm/tests/test_ucx_config.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests +copying build/lib/distributed/comm/tests/test_tcp.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests +copying build/lib/distributed/comm/tests/test_ws.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests +copying build/lib/distributed/comm/tests/test_ucx.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests +copying build/lib/distributed/comm/tests/test_comms.py -> build/bdist.linux-x86_64/wheel/./distributed/comm/tests +copying build/lib/distributed/comm/ucx.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/tcp.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/comm/ws.py -> build/bdist.linux-x86_64/wheel/./distributed/comm +copying build/lib/distributed/_asyncio.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/recreate_tasks.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/_version.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/versions.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/collections.py -> build/bdist.linux-x86_64/wheel/./distributed +copying build/lib/distributed/queues.py -> build/bdist.linux-x86_64/wheel/./distributed +creating build/bdist.linux-x86_64/wheel/distributed/cli +copying build/lib/distributed/cli/dask_worker.py -> build/bdist.linux-x86_64/wheel/./distributed/cli +copying build/lib/distributed/cli/dask_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/cli +copying build/lib/distributed/cli/dask_scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/cli +copying build/lib/distributed/cli/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/cli +creating build/bdist.linux-x86_64/wheel/distributed/cli/tests +copying build/lib/distributed/cli/tests/test_dask_worker.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests +copying build/lib/distributed/cli/tests/__init__.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests +copying build/lib/distributed/cli/tests/test_dask_scheduler.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests +copying build/lib/distributed/cli/tests/test_dask_ssh.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests +copying build/lib/distributed/cli/tests/test_tls_cli.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests +copying build/lib/distributed/cli/tests/test_dask_spec.py -> build/bdist.linux-x86_64/wheel/./distributed/cli/tests +copying build/lib/distributed/cli/utils.py -> build/bdist.linux-x86_64/wheel/./distributed/cli +copying build/lib/distributed/cli/dask_spec.py -> build/bdist.linux-x86_64/wheel/./distributed/cli running install_egg_info Copying distributed.egg-info to build/bdist.linux-x86_64/wheel/./distributed-2024.5.2.egg-info running install_scripts creating build/bdist.linux-x86_64/wheel/distributed-2024.5.2.dist-info/WHEEL -creating '/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/.tmp-jadwj0ot/distributed-2024.5.2-py3-none-any.whl' and adding 'build/bdist.linux-x86_64/wheel' to it +creating '/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/.tmp-lpusj2qc/distributed-2024.5.2-py3-none-any.whl' and adding 'build/bdist.linux-x86_64/wheel' to it adding 'distributed/__init__.py' adding 'distributed/_asyncio.py' adding 'distributed/_concurrent_futures_thread.py' @@ -3378,7 +3414,7 @@ cachedir: .pytest_cache rootdir: /build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build configfile: pyproject.toml -plugins: anyio-4.4.0, typeguard-4.3.0, rerunfailures-12.0, timeout-2.3.1 +plugins: rerunfailures-12.0, typeguard-4.3.0, timeout-2.3.1, anyio-4.4.0 timeout: 300.0s timeout method: signal timeout func_only: False @@ -3480,6 +3516,7 @@ distributed/diagnostics/tests/test_nanny_plugin.py::test_register_idempotent_plugin PASSED [ 3%] distributed/diagnostics/tests/test_nanny_plugin.py::test_register_non_idempotent_plugin PASSED [ 3%] distributed/diagnostics/tests/test_nanny_plugin.py::test_register_plugin_with_idempotent_keyword_is_deprecated PASSED [ 3%] +distributed/diagnostics/tests/test_progress.py::test_many_Progress RERUN [ 3%] distributed/diagnostics/tests/test_progress.py::test_many_Progress PASSED [ 3%] distributed/diagnostics/tests/test_progress.py::test_multiprogress PASSED [ 3%] distributed/diagnostics/tests/test_progress.py::test_multiprogress_cancel PASSED [ 3%] @@ -3987,7 +4024,7 @@ distributed/shuffle/tests/test_rechunk.py::test_rechunk_4d[False] PASSED [ 19%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_expand PASSED [ 19%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_expand2 PASSED [ 19%] -distributed/shuffle/tests/test_rechunk.py::test_rechunk_method PASSED [ 19%] +distributed/shuffle/tests/test_rechunk.py::test_rechunk_method FAILED [ 19%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_blockshape PASSED [ 19%] distributed/shuffle/tests/test_rechunk.py::test_dtype PASSED [ 19%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_dict PASSED [ 19%] @@ -4025,9 +4062,9 @@ distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x0-chunks0] PASSED [ 20%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x1-chunks1] PASSED [ 20%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x2-chunks2] PASSED [ 20%] -distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] PASSED [ 20%] +distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] FAILED [ 20%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x4-chunks4] PASSED [ 20%] -distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] PASSED [ 20%] +distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] FAILED [ 20%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x6-chunks6] PASSED [ 20%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x7-chunks7] PASSED [ 20%] distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x8-chunks8] PASSED [ 20%] @@ -4250,7 +4287,7 @@ distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[False-resumed] PASSED [ 27%] distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[False-cancelled] PASSED [ 27%] distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-resumed] PASSED [ 27%] -distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] PASSED [ 27%] +distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] FAILED [ 27%] distributed/tests/test_cancelled_state.py::test_workerstate_executing_to_executing[executing] PASSED [ 27%] distributed/tests/test_cancelled_state.py::test_workerstate_executing_to_executing[long-running] PASSED [ 27%] distributed/tests/test_cancelled_state.py::test_workerstate_flight_to_flight PASSED [ 27%] @@ -4471,7 +4508,7 @@ distributed/tests/test_client.py::test_run_coroutine PASSED [ 34%] distributed/tests/test_client.py::test_run_coroutine_sync PASSED [ 34%] distributed/tests/test_client.py::test_run_exception PASSED [ 34%] -distributed/tests/test_client.py::test_run_rpc_error PASSED [ 34%] +distributed/tests/test_client.py::test_run_rpc_error FAILED [ 34%] distributed/tests/test_client.py::test_diagnostic_ui PASSED [ 34%] distributed/tests/test_client.py::test_diagnostic_nbytes_sync PASSED [ 34%] distributed/tests/test_client.py::test_diagnostic_nbytes PASSED [ 34%] @@ -5070,7 +5107,7 @@ distributed/tests/test_priorities.py::test_annotate_persist[queue on scheduler] PASSED [ 52%] distributed/tests/test_priorities.py::test_repeated_persists_same_priority[queue on worker] PASSED [ 52%] distributed/tests/test_priorities.py::test_repeated_persists_same_priority[queue on scheduler] PASSED [ 52%] -distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] PASSED [ 52%] +distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] FAILED [ 52%] distributed/tests/test_priorities.py::test_last_in_first_out[queue on scheduler] PASSED [ 52%] distributed/tests/test_profile.py::test_basic PASSED [ 52%] distributed/tests/test_profile.py::test_basic_low_level SKIPPED (could @@ -6576,27 +6613,2068 @@ distributed/tests/test_worker_state_machine.py::test_remove_worker_while_in_fetch PASSED [ 99%] distributed/tests/test_worker_state_machine.py::test_remove_worker_unknown PASSED [100%] +=================================== FAILURES =================================== +_____________________________ test_rechunk_method ______________________________ + +fut = , timeout = 29.72892400296405 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut +E asyncio.exceptions.CancelledError + +distributed/utils.py:1960: CancelledError + +The above exception was the direct cause of the following exception: + + async def async_fn(): + result = None + with dask.config.set(config): + async with ( + _cluster_factory() as (s, workers), + _client_factory(s) as c, + ): + args = [s] + workers + if c is not None: + args = [c] + args + try: + coro = func(*args, *outer_args, **kwargs) + task = asyncio.create_task(coro) + coro2 = utils_wait_for( + asyncio.shield(task), timeout=deadline.remaining + ) +> result = await coro2 + +distributed/utils_test.py:1011: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError + +During handling of the above exception, another exception occurred: + +fut = ._..test_func..async_fn at 0x7f220244f4d0> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1960: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1031: in async_fn + await asyncio.sleep(0.01) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +delay = 0.01, result = None + + async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result + + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: +> return await future +E asyncio.exceptions.CancelledError + +/usr/lib/python3.12/asyncio/tasks.py:665: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.12/contextlib.py:81: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.12/contextlib.py:81: in inner + return func(*args, **kwds) +distributed/utils_test.py:1102: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:378: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.12/asyncio/runners.py:194: in run + return runner.run(main) +/usr/lib/python3.12/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.12/asyncio/base_events.py:687: in run_until_complete + return future.result() +distributed/utils_test.py:375: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1099: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError +----------------------------- Captured stdout call ----------------------------- +Dumped cluster state to test_cluster_dump/test_rechunk_method.yaml +----------------------------- Captured stderr call ----------------------------- +2024-10-22 06:38:13,077 - distributed.scheduler - INFO - State start +2024-10-22 06:38:13,090 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:35785 +2024-10-22 06:38:13,090 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:32813/status +2024-10-22 06:38:13,091 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 06:38:13,146 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:43603 +2024-10-22 06:38:13,146 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:43603 +2024-10-22 06:38:13,147 - distributed.worker - INFO - Worker name: 0 +2024-10-22 06:38:13,147 - distributed.worker - INFO - dashboard at: 127.0.0.1:43003 +2024-10-22 06:38:13,147 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:35785 +2024-10-22 06:38:13,147 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:38:13,147 - distributed.worker - INFO - Threads: 1 +2024-10-22 06:38:13,147 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:38:13,147 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-41ycmmlp +2024-10-22 06:38:13,147 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:38:13,148 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:36355 +2024-10-22 06:38:13,148 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:36355 +2024-10-22 06:38:13,161 - distributed.worker - INFO - Worker name: 1 +2024-10-22 06:38:13,161 - distributed.worker - INFO - dashboard at: 127.0.0.1:40887 +2024-10-22 06:38:13,161 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:35785 +2024-10-22 06:38:13,161 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:38:13,161 - distributed.worker - INFO - Threads: 2 +2024-10-22 06:38:13,161 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:38:13,161 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-pnf_e471 +2024-10-22 06:38:13,161 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:38:13,215 - distributed.scheduler - INFO - Register worker +2024-10-22 06:38:13,238 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:43603 +2024-10-22 06:38:13,238 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:33384 +2024-10-22 06:38:13,239 - distributed.scheduler - INFO - Register worker +2024-10-22 06:38:13,265 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:36355 +2024-10-22 06:38:13,265 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:33392 +2024-10-22 06:38:13,266 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:38:13,266 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:38:13,267 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:35785 +2024-10-22 06:38:13,267 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:38:13,268 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:35785 +2024-10-22 06:38:13,268 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:38:13,268 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:35785 +2024-10-22 06:38:13,277 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:35785 +2024-10-22 06:38:13,315 - distributed.scheduler - INFO - Receive client connection: Client-dd6be4fc-8fca-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:38:13,325 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:33398 +2024-10-22 06:38:14,627 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0041321168557503bfe77cc4c2fa8ad5 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 2, 1, 2, 1, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:14,756 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 88d51be1b20da5590bf384cab709a13c initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 2, 0, 1, 1, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:14,887 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d6585e17cb212ca77cc0912fa727b367 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 1, 2, 2, 2, 1, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:15,056 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 17165189af0e28bda349675e83aa4d40 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 1, 1, 2, 2, 1, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:16,300 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e39985504976966b80ba04bba3046983 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 1, 0, 2, 1, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:16,466 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 00ee5e70b3d35104959c482bab9c0218 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 2, 1, 2, 1, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:16,655 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 473b2aff313c060220c21ae000400c5e initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 2, 2, 2, 1, 1, 1) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:16,739 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 316f4fc008c10683a5956fc2c5eb530c initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 2, 0, 1, 1, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:17,008 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ff3a74af796c0c0c0bac8500f84a9b80 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 1, 1, 2, 1, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:17,040 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 45b61ea6e3ad26f4d76d17ad5bc53779 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 1, 2, 2, 1, 1, 1) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:17,216 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 32b027e1b4dc3e43d2fa6f8c98119678 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 1, 0, 2, 1, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:17,262 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 1fbb7727ee6f46056e97d95d524fb463 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 2, 1, 1, 1, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:17,575 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a802f8a55cac32673836f4a1905c0c9f initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 2, 0, 1, 1, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:17,698 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 72f360d254fb62e177e2e195b3766722 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 1, 2, 1, 1, 1, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:17,787 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3551bf46927be8ed9ead691352051359 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 2, 2, 2, 2, 2, 1) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:17,902 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 71bb09e1ea92dcc5ff33bc820a96326a initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 1, 2, 0, 1, 1, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:19,396 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ed7b1e59d6dd11800295684a812a06a initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 2, 2, 1, 1, 1, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:19,877 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 60c8256a13826c1d697ed71561097ee6 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 1, 2, 1, 1, 1, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:19,943 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f94b3b86bf8ff017af1b7b4dcf4b6518 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 1, 1, 1, 2, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:20,084 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle de1c3beca688f341f59797f18c823496 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 1, 2, 2, 1, 1, 2, 1) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:20,156 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a527bfb95948458ee116d8f15fd10351 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 1, 0, 1, 2, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:20,478 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cd8bcdfa6ff3c5f40ab1da84d00cf452 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 1, 1, 2, 1, 1, 1, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:24,268 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 43e10d58d11aae5ceb65e03b73aaafd7 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 0, 2, 2, 2, 0, 2) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:24,709 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0041321168557503bfe77cc4c2fa8ad5 deactivated due to stimulus 'task-finished-1729528704.705562' +2024-10-22 06:38:25,177 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 71c5b020e22591132efe133e3709d1f4 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 0, 1, 2, 2, 0, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:25,877 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 88d51be1b20da5590bf384cab709a13c deactivated due to stimulus 'task-finished-1729528705.7707915' +2024-10-22 06:38:26,923 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 87d19895dd066470e26dc64fd4cb9582 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 0, 2, 2, 1, 0, 2) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:27,239 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0a07c2129781bf3bf34aa97f8ebc2023 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 0, 1, 2, 1, 0, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:27,565 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b82ed302966a2d3e696a15394198e6c3 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 0, 2, 1, 2, 0, 2) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:28,620 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a199acda10ad5b63f165cc2bca5cd9e3 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 0, 1, 1, 2, 0, 1) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:30,460 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ae340cebcfd3c13c9a8b488237d4a541 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 2, 0, 0, 2, 2, 0, 0) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:30,474 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a607bd2bdd49a31c1cf81f58192aee43 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 1, 0, 2, 1, 1, 0, 2) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:31,264 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d6585e17cb212ca77cc0912fa727b367 deactivated due to stimulus 'task-finished-1729528711.2612853' +2024-10-22 06:38:31,715 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 43e10d58d11aae5ceb65e03b73aaafd7 deactivated due to stimulus 'task-finished-1729528711.6895661' +2024-10-22 06:38:32,146 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f4fdfbabe2648c343bace34c4f069508 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 1, 0, 0, 2, 1, 0, 0) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:34,773 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b4f1a217afa2b30d17d7658f7ec52e16 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 2, 0, 0, 1, 2, 0, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:34,866 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 17165189af0e28bda349675e83aa4d40 deactivated due to stimulus 'task-finished-1729528714.8628566' +2024-10-22 06:38:35,536 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e39985504976966b80ba04bba3046983 deactivated due to stimulus 'task-finished-1729528715.5335236' +2024-10-22 06:38:35,836 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ae340cebcfd3c13c9a8b488237d4a541 deactivated due to stimulus 'task-finished-1729528715.7566056' +2024-10-22 06:38:36,109 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a53734a94dad88ff2866c896818900a3 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 0, 2, 2, 2, 0, 2, 2) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:36,262 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 71c5b020e22591132efe133e3709d1f4 deactivated due to stimulus 'task-finished-1729528716.2057667' +2024-10-22 06:38:36,263 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 25148c801305cb4d040d00a470d3a564 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 0, 2, 1, 2, 0, 2, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:37,418 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 021652c22b78524e5ffb746fbb813b82 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 0, 1, 2, 2, 0, 1, 2) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:37,877 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 6b0fa3f0145e9021527b443a2bd9c9cb initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 0, 2, 2, 1, 0, 2, 2) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:38,210 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 8a32e2d0e79724374a4d5dea34362e3d initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 0, 1, 1, 2, 0, 1, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:38,879 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0418c3198fb3bc1161ac90636208fd5b initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 0, 2, 1, 1, 0, 2, 1) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:39,834 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b3cbdc229060e31562dac51c85d8502b initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 0, 2, 0, 2, 0, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:40,970 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 473b2aff313c060220c21ae000400c5e deactivated due to stimulus 'task-finished-1729528720.9155438' +2024-10-22 06:38:42,018 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d26b76a95658f00d8c2e21974fb81b88 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 2, 0, 1, 0, 2, 0, 1, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:42,097 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ed624c55ccda318bb49f0905f3a99ef initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 0, 1, 2, 1, 0, 1, 2) executed on worker tcp://127.0.0.1:43603 +2024-10-22 06:38:42,562 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 76bb3ecca0c28e4b338288992a9ba670 initialized by task ('rechunk-transfer-3944b7a19999afcc7f2f0bba85002ac1', 1, 0, 2, 0, 1, 0, 2, 0) executed on worker tcp://127.0.0.1:36355 +2024-10-22 06:38:42,771 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a53734a94dad88ff2866c896818900a3 deactivated due to stimulus 'task-finished-1729528722.7543442' +2024-10-22 06:41:20,903 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 158.28s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:41:20,938 - distributed.core - INFO - Event loop was unresponsive in Worker for 158.12s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:41:20,964 - distributed.core - INFO - Event loop was unresponsive in Worker for 158.13s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:41:21,112 - distributed.scheduler - INFO - Remove client Client-dd6be4fc-8fca-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:41:21,141 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f94b3b86bf8ff017af1b7b4dcf4b6518 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,159 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a607bd2bdd49a31c1cf81f58192aee43 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,160 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 32b027e1b4dc3e43d2fa6f8c98119678 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,174 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 72f360d254fb62e177e2e195b3766722 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,208 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a802f8a55cac32673836f4a1905c0c9f deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,224 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b82ed302966a2d3e696a15394198e6c3 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,238 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a527bfb95948458ee116d8f15fd10351 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,240 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 45b61ea6e3ad26f4d76d17ad5bc53779 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,254 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle de1c3beca688f341f59797f18c823496 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,256 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 00ee5e70b3d35104959c482bab9c0218 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,271 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b4f1a217afa2b30d17d7658f7ec52e16 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,271 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 316f4fc008c10683a5956fc2c5eb530c deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,272 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 87d19895dd066470e26dc64fd4cb9582 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,286 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ff3a74af796c0c0c0bac8500f84a9b80 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,301 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b3cbdc229060e31562dac51c85d8502b deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,302 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 021652c22b78524e5ffb746fbb813b82 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,303 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f4fdfbabe2648c343bace34c4f069508 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,317 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 71bb09e1ea92dcc5ff33bc820a96326a deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,318 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cd8bcdfa6ff3c5f40ab1da84d00cf452 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,320 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 6b0fa3f0145e9021527b443a2bd9c9cb deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,335 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0a07c2129781bf3bf34aa97f8ebc2023 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,349 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 25148c801305cb4d040d00a470d3a564 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,350 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ed624c55ccda318bb49f0905f3a99ef deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,352 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle a199acda10ad5b63f165cc2bca5cd9e3 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,366 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d26b76a95658f00d8c2e21974fb81b88 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,367 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 8a32e2d0e79724374a4d5dea34362e3d deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,384 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 76bb3ecca0c28e4b338288992a9ba670 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,398 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0418c3198fb3bc1161ac90636208fd5b deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,400 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 60c8256a13826c1d697ed71561097ee6 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,434 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ed7b1e59d6dd11800295684a812a06a deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,451 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3551bf46927be8ed9ead691352051359 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,467 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 1fbb7727ee6f46056e97d95d524fb463 deactivated due to stimulus 'remove-client-1729528881.1121778' +2024-10-22 06:41:21,498 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:33398; closing. +2024-10-22 06:41:21,514 - distributed.scheduler - INFO - Remove client Client-dd6be4fc-8fca-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:41:21,678 - distributed.scheduler - INFO - Close client connection: Client-dd6be4fc-8fca-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:41:21,678 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:43603. Reason: worker-close +2024-10-22 06:41:21,678 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:36355. Reason: worker-close +2024-10-22 06:41:21,720 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:41:21,789 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:41:21,790 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:41:21,790 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:41:22,015 - distributed.shuffle._comms - ERROR - SchedulerShuffleState<87d19895dd066470e26dc64fd4cb9582[40]> forgotten +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 203, in _send + self.raise_if_closed() + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 266, in raise_if_closed + raise self._exception +RuntimeError: SchedulerShuffleState<87d19895dd066470e26dc64fd4cb9582[40]> forgotten +2024-10-22 06:41:22,078 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: '45b61ea6e3ad26f4d76d17ad5bc53779' +2024-10-22 06:41:22,094 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: '316f4fc008c10683a5956fc2c5eb530c' +2024-10-22 06:41:22,184 - distributed.core - ERROR - Exception while handling op shuffle_receive +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '45b61ea6e3ad26f4d76d17ad5bc53779' +2024-10-22 06:41:22,224 - distributed.core - ERROR - Exception while handling op shuffle_receive +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '316f4fc008c10683a5956fc2c5eb530c' +2024-10-22 06:41:22,279 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: '87d19895dd066470e26dc64fd4cb9582' +2024-10-22 06:41:22,316 - distributed.shuffle._comms - ERROR - '45b61ea6e3ad26f4d76d17ad5bc53779' +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 204, in _send + return await self.rpc(address).shuffle_receive( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '45b61ea6e3ad26f4d76d17ad5bc53779' +2024-10-22 06:41:22,354 - distributed.shuffle._comms - ERROR - '316f4fc008c10683a5956fc2c5eb530c' +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 204, in _send + return await self.rpc(address).shuffle_receive( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '316f4fc008c10683a5956fc2c5eb530c' +2024-10-22 06:41:22,451 - distributed.core - ERROR - Exception while handling op shuffle_receive +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '87d19895dd066470e26dc64fd4cb9582' +2024-10-22 06:41:22,507 - distributed.shuffle._comms - ERROR - '87d19895dd066470e26dc64fd4cb9582' +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 204, in _send + return await self.rpc(address).shuffle_receive( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '87d19895dd066470e26dc64fd4cb9582' +2024-10-22 06:41:22,544 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:33384; closing. +2024-10-22 06:41:22,554 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729528882.5539432') +2024-10-22 06:41:22,554 - distributed.core - INFO - Connection to tcp://127.0.0.1:35785 has been closed. +2024-10-22 06:41:22,570 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:33392; closing. +2024-10-22 06:41:22,571 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729528882.5709538') +2024-10-22 06:41:22,571 - distributed.scheduler - INFO - Lost all workers +2024-10-22 06:41:22,571 - distributed.core - INFO - Connection to tcp://127.0.0.1:35785 has been closed. +2024-10-22 06:41:22,604 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 06:41:22,613 - distributed.scheduler - INFO - Scheduler closing all comms +__________ test_rechunk_with_partially_unknown_dimension[x3-chunks3] ___________ + +fut = , timeout = 29.69062902405858 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut +E asyncio.exceptions.CancelledError + +distributed/utils.py:1960: CancelledError + +The above exception was the direct cause of the following exception: + + async def async_fn(): + result = None + with dask.config.set(config): + async with ( + _cluster_factory() as (s, workers), + _client_factory(s) as c, + ): + args = [s] + workers + if c is not None: + args = [c] + args + try: + coro = func(*args, *outer_args, **kwargs) + task = asyncio.create_task(coro) + coro2 = utils_wait_for( + asyncio.shield(task), timeout=deadline.remaining + ) +> result = await coro2 + +distributed/utils_test.py:1011: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError + +During handling of the above exception, another exception occurred: + +fut = ._..test_func..async_fn at 0x7f220244f870> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1960: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1031: in async_fn + await asyncio.sleep(0.01) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +delay = 0.01, result = None + + async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result + + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: +> return await future +E asyncio.exceptions.CancelledError + +/usr/lib/python3.12/asyncio/tasks.py:665: CancelledError + +The above exception was the direct cause of the following exception: + +args = () +kwds = {'chunks': (None, 5), 'x': dask.array} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.12/contextlib.py:81: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.12/contextlib.py:81: in inner + return func(*args, **kwds) +distributed/utils_test.py:1102: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:378: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.12/asyncio/runners.py:194: in run + return runner.run(main) +/usr/lib/python3.12/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.12/asyncio/base_events.py:687: in run_until_complete + return future.result() +distributed/utils_test.py:375: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1099: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError +----------------------------- Captured stdout call ----------------------------- +Dumped cluster state to test_cluster_dump/test_rechunk_with_partially_unknown_dimension.yaml +----------------------------- Captured stderr call ----------------------------- +2024-10-22 06:42:49,821 - distributed.scheduler - INFO - State start +2024-10-22 06:42:49,845 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:42749 +2024-10-22 06:42:49,845 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44325/status +2024-10-22 06:42:49,846 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 06:42:49,888 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:40781 +2024-10-22 06:42:49,888 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:40781 +2024-10-22 06:42:49,888 - distributed.worker - INFO - Worker name: 0 +2024-10-22 06:42:49,888 - distributed.worker - INFO - dashboard at: 127.0.0.1:44133 +2024-10-22 06:42:49,888 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:42749 +2024-10-22 06:42:49,888 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:42:49,888 - distributed.worker - INFO - Threads: 1 +2024-10-22 06:42:49,888 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:42:49,897 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-kt8ilrzi +2024-10-22 06:42:49,897 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:42:49,899 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:37287 +2024-10-22 06:42:49,899 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:37287 +2024-10-22 06:42:49,899 - distributed.worker - INFO - Worker name: 1 +2024-10-22 06:42:49,899 - distributed.worker - INFO - dashboard at: 127.0.0.1:37085 +2024-10-22 06:42:49,899 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:42749 +2024-10-22 06:42:49,900 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:42:49,900 - distributed.worker - INFO - Threads: 2 +2024-10-22 06:42:49,900 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:42:49,900 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-kb3p4n4x +2024-10-22 06:42:49,900 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:42:49,980 - distributed.scheduler - INFO - Register worker +2024-10-22 06:42:50,017 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:40781 +2024-10-22 06:42:50,017 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34508 +2024-10-22 06:42:50,018 - distributed.scheduler - INFO - Register worker +2024-10-22 06:42:50,043 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:37287 +2024-10-22 06:42:50,043 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34522 +2024-10-22 06:42:50,044 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:42:50,044 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:42:50,054 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:42749 +2024-10-22 06:42:50,054 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:42:50,054 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:42749 +2024-10-22 06:42:50,054 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:42:50,055 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:42749 +2024-10-22 06:42:50,056 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:42749 +2024-10-22 06:42:50,095 - distributed.scheduler - INFO - Receive client connection: Client-8264f701-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:42:50,115 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34524 +2024-10-22 06:42:50,551 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 17261aa4df64de4bbec9ec8f778b6988 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-17261aa4df64de4bbec9ec8f778b6988' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:50,552 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e59664493e67b4922535dece242d1aab initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-e59664493e67b4922535dece242d1aab' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:42:50,552 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bce680412476b294e188ce6816b7e2cd initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-bce680412476b294e188ce6816b7e2cd' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:50,960 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d818f5d4a45fa1d93be4d23875883b80 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-d818f5d4a45fa1d93be4d23875883b80' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:50,960 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01db6bbc0c226a1ef7d4a6b3f462f6bb initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-01db6bbc0c226a1ef7d4a6b3f462f6bb' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:50,970 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 31fcedb5edb6afb1da8e949ae718fd04 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-31fcedb5edb6afb1da8e949ae718fd04' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:42:51,319 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01e1fb53041e5fdb0d29bbe0a220a186 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-01e1fb53041e5fdb0d29bbe0a220a186' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:42:51,320 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ddb3383fd4d500ceb54cf08b5b6afae1 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-ddb3383fd4d500ceb54cf08b5b6afae1' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:51,320 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c2ee4274eae62a3e133ce55a83985341 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-c2ee4274eae62a3e133ce55a83985341' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:52,042 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e59664493e67b4922535dece242d1aab deactivated due to stimulus 'task-finished-1729528972.014734' +2024-10-22 06:42:52,205 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 17261aa4df64de4bbec9ec8f778b6988 deactivated due to stimulus 'task-finished-1729528972.123714' +2024-10-22 06:42:52,499 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc599f874b11835571ee6bcce1edc1d8 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-cc599f874b11835571ee6bcce1edc1d8' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:52,518 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bce680412476b294e188ce6816b7e2cd deactivated due to stimulus 'task-finished-1729528972.470268' +2024-10-22 06:42:52,560 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d818f5d4a45fa1d93be4d23875883b80 deactivated due to stimulus 'task-finished-1729528972.5204387' +2024-10-22 06:42:52,790 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01db6bbc0c226a1ef7d4a6b3f462f6bb deactivated due to stimulus 'task-finished-1729528972.7587216' +2024-10-22 06:42:53,195 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 31fcedb5edb6afb1da8e949ae718fd04 deactivated due to stimulus 'task-finished-1729528973.193394' +2024-10-22 06:42:53,230 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e765252ecfc2b6ffd5668c3b385a3174 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-e765252ecfc2b6ffd5668c3b385a3174' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:53,270 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ddb3383fd4d500ceb54cf08b5b6afae1 deactivated due to stimulus 'task-finished-1729528973.2271366' +2024-10-22 06:42:53,408 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 84386ba0af6f45a3973ad458a14d56c5 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-84386ba0af6f45a3973ad458a14d56c5' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:53,734 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c81c046e65c84ae49e04fd567198f65c initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-c81c046e65c84ae49e04fd567198f65c' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:53,946 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c2ee4274eae62a3e133ce55a83985341 deactivated due to stimulus 'task-finished-1729528973.8805308' +2024-10-22 06:42:54,212 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc599f874b11835571ee6bcce1edc1d8 deactivated due to stimulus 'task-finished-1729528974.0711687' +2024-10-22 06:42:54,345 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01e1fb53041e5fdb0d29bbe0a220a186 deactivated due to stimulus 'task-finished-1729528974.2910295' +2024-10-22 06:42:54,347 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d739332393d2d6dae0f074f82be858b3 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-d739332393d2d6dae0f074f82be858b3' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:54,444 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e765252ecfc2b6ffd5668c3b385a3174 deactivated due to stimulus 'task-finished-1729528974.387503' +2024-10-22 06:42:54,803 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 59881a6059b5a68660aef4b1433f259a initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-59881a6059b5a68660aef4b1433f259a' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:42:55,236 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e67f678006c2dfb420382b0653e7e278 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-e67f678006c2dfb420382b0653e7e278' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:55,461 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b79d0d8845743010f9e3088a0beb18d1 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-b79d0d8845743010f9e3088a0beb18d1' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:55,463 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c81c046e65c84ae49e04fd567198f65c deactivated due to stimulus 'task-finished-1729528975.446447' +2024-10-22 06:42:55,802 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 84386ba0af6f45a3973ad458a14d56c5 deactivated due to stimulus 'task-finished-1729528975.7701173' +2024-10-22 06:42:55,888 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f9160caf490b8329e7669ecbd1ac041e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-f9160caf490b8329e7669ecbd1ac041e' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:56,117 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d739332393d2d6dae0f074f82be858b3 deactivated due to stimulus 'task-finished-1729528976.1140413' +2024-10-22 06:42:56,195 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 59881a6059b5a68660aef4b1433f259a deactivated due to stimulus 'task-finished-1729528976.1796134' +2024-10-22 06:42:56,234 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e67f678006c2dfb420382b0653e7e278 deactivated due to stimulus 'task-finished-1729528976.2199976' +2024-10-22 06:42:56,242 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec3af6e3853fbf1e077a45ec7a358cac initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-ec3af6e3853fbf1e077a45ec7a358cac' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:42:56,559 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec792f9b679daef65c0ec6556b5e9c9f initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-ec792f9b679daef65c0ec6556b5e9c9f' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:56,732 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle df475960043b4dab9b72a8cebe24fbf0 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-df475960043b4dab9b72a8cebe24fbf0' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:56,919 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b79d0d8845743010f9e3088a0beb18d1 deactivated due to stimulus 'task-finished-1729528976.757466' +2024-10-22 06:42:57,290 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f9160caf490b8329e7669ecbd1ac041e deactivated due to stimulus 'task-finished-1729528977.2782779' +2024-10-22 06:42:57,321 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e77aa0a14aa79e0a383a0b353abba873 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-e77aa0a14aa79e0a383a0b353abba873' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:58,207 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec3af6e3853fbf1e077a45ec7a358cac deactivated due to stimulus 'task-finished-1729528978.0769322' +2024-10-22 06:42:58,453 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle df475960043b4dab9b72a8cebe24fbf0 deactivated due to stimulus 'task-finished-1729528978.450613' +2024-10-22 06:42:58,571 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 714ef9e25e8b5c1d5b3451d982311c2e initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-714ef9e25e8b5c1d5b3451d982311c2e' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:59,016 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec792f9b679daef65c0ec6556b5e9c9f deactivated due to stimulus 'task-finished-1729528978.9305575' +2024-10-22 06:42:59,074 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b8fcb44d156414c630e1d6ff0df77c1b initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-b8fcb44d156414c630e1d6ff0df77c1b' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:42:59,361 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 57d52a42fcd6be719e65e6f9840d0cdb initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-57d52a42fcd6be719e65e6f9840d0cdb' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:59,470 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b9c215f5d25889bd6c1d4a0fb35eb468 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-b9c215f5d25889bd6c1d4a0fb35eb468' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:42:59,682 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e77aa0a14aa79e0a383a0b353abba873 deactivated due to stimulus 'task-finished-1729528979.571674' +2024-10-22 06:42:59,699 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f385202a9df8dca26b51e3b27030056b initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-f385202a9df8dca26b51e3b27030056b' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:00,420 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 714ef9e25e8b5c1d5b3451d982311c2e deactivated due to stimulus 'task-finished-1729528980.362628' +2024-10-22 06:43:00,651 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cf4000d0bbe7b2713a9b81d90981fe9b initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-cf4000d0bbe7b2713a9b81d90981fe9b' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:00,678 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 1164a44fb53f1d99c26d483a9f7618c2 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-1164a44fb53f1d99c26d483a9f7618c2' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:00,763 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 57d52a42fcd6be719e65e6f9840d0cdb deactivated due to stimulus 'task-finished-1729528980.713456' +2024-10-22 06:43:01,206 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3eabd736c25cbfb05d814f9f3d9bfc98 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-3eabd736c25cbfb05d814f9f3d9bfc98' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:01,210 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0fa082cb36216f52a7836b89276b16a9 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-0fa082cb36216f52a7836b89276b16a9' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:01,347 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f385202a9df8dca26b51e3b27030056b deactivated due to stimulus 'task-finished-1729528981.3423245' +2024-10-22 06:43:01,509 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b9c215f5d25889bd6c1d4a0fb35eb468 deactivated due to stimulus 'task-finished-1729528981.5068707' +2024-10-22 06:43:01,554 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cf4000d0bbe7b2713a9b81d90981fe9b deactivated due to stimulus 'task-finished-1729528981.5318544' +2024-10-22 06:43:01,567 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 11f1f6ff4fd0902dbfb6a01bfdb5359d initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-11f1f6ff4fd0902dbfb6a01bfdb5359d' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:01,688 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b8fcb44d156414c630e1d6ff0df77c1b deactivated due to stimulus 'task-finished-1729528981.6691968' +2024-10-22 06:43:01,835 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3eabd736c25cbfb05d814f9f3d9bfc98 deactivated due to stimulus 'task-finished-1729528981.8277154' +2024-10-22 06:43:01,896 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 4a20b1bca8a7c0bc5fe798d5d474efe7 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-4a20b1bca8a7c0bc5fe798d5d474efe7' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:02,122 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 8f6377159296c2aea721b80be7444955 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-8f6377159296c2aea721b80be7444955' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:02,174 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 1164a44fb53f1d99c26d483a9f7618c2 deactivated due to stimulus 'task-finished-1729528982.1533895' +2024-10-22 06:43:02,303 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 23c3e0a6318f8719413afcb7fb1c165c initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-23c3e0a6318f8719413afcb7fb1c165c' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:02,333 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2037175d04234fb783dd45ef54a900ac initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-2037175d04234fb783dd45ef54a900ac' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:02,682 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0cc7ac64edd3e809e1ee98f4e84d65e4 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-0cc7ac64edd3e809e1ee98f4e84d65e4' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:02,787 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle fd6cd470d74c25edf76b3f4dc74c12c0 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-fd6cd470d74c25edf76b3f4dc74c12c0' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:02,788 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 4a20b1bca8a7c0bc5fe798d5d474efe7 deactivated due to stimulus 'task-finished-1729528982.78543' +2024-10-22 06:43:03,000 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 8f6377159296c2aea721b80be7444955 deactivated due to stimulus 'task-finished-1729528982.9971752' +2024-10-22 06:43:03,017 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 11f1f6ff4fd0902dbfb6a01bfdb5359d deactivated due to stimulus 'task-finished-1729528982.9979885' +2024-10-22 06:43:03,116 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 23c3e0a6318f8719413afcb7fb1c165c deactivated due to stimulus 'task-finished-1729528983.113419' +2024-10-22 06:43:03,116 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0fa082cb36216f52a7836b89276b16a9 deactivated due to stimulus 'task-finished-1729528983.1135314' +2024-10-22 06:43:03,243 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 92126407b7617e8e1154192b00785cb2 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-92126407b7617e8e1154192b00785cb2' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:03,244 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 204b3c4c3770c441622961008d21fe35 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-204b3c4c3770c441622961008d21fe35' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:03,514 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2037175d04234fb783dd45ef54a900ac deactivated due to stimulus 'task-finished-1729528983.5131118' +2024-10-22 06:43:03,615 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0cc7ac64edd3e809e1ee98f4e84d65e4 deactivated due to stimulus 'task-finished-1729528983.6038828' +2024-10-22 06:43:03,779 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle fd6cd470d74c25edf76b3f4dc74c12c0 deactivated due to stimulus 'task-finished-1729528983.7774303' +2024-10-22 06:43:03,802 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 204b3c4c3770c441622961008d21fe35 deactivated due to stimulus 'task-finished-1729528983.801175' +2024-10-22 06:43:03,888 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 92126407b7617e8e1154192b00785cb2 deactivated due to stimulus 'task-finished-1729528983.8855662' +2024-10-22 06:43:06,022 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 806d3b72ee98acf548c7fd2bbb441c36 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-806d3b72ee98acf548c7fd2bbb441c36' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:06,023 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bc4eab2e4c69f1370607a11f2a9c4392 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-bc4eab2e4c69f1370607a11f2a9c4392' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:06,142 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d07954b281dba89624cbc6c762797e1f initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-d07954b281dba89624cbc6c762797e1f' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:07,952 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bcb2f010c2aeaa2bad84b81d2684fa7c initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-bcb2f010c2aeaa2bad84b81d2684fa7c' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:07,953 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 38071ec7b016834cb180426b1b3240a7 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-38071ec7b016834cb180426b1b3240a7' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:08,129 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle aa36ff673eca80feec08ff85465e098e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-aa36ff673eca80feec08ff85465e098e' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:08,824 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3c148173ebc946767fe147506184e5d5 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-3c148173ebc946767fe147506184e5d5' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:08,867 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 444ea2ff016bfbcb8c2079368d20079f initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-444ea2ff016bfbcb8c2079368d20079f' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:08,868 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0d7e0559a60e99a8106e3804f6ade934 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-0d7e0559a60e99a8106e3804f6ade934' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:09,667 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 806d3b72ee98acf548c7fd2bbb441c36 deactivated due to stimulus 'task-finished-1729528989.6654696' +2024-10-22 06:43:09,744 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c97865205e833b24dffa05881b7a9355 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-c97865205e833b24dffa05881b7a9355' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:09,767 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d07954b281dba89624cbc6c762797e1f deactivated due to stimulus 'task-finished-1729528989.7414484' +2024-10-22 06:43:10,460 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 38071ec7b016834cb180426b1b3240a7 deactivated due to stimulus 'task-finished-1729528990.4580576' +2024-10-22 06:43:10,803 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle aa36ff673eca80feec08ff85465e098e deactivated due to stimulus 'task-finished-1729528990.7988675' +2024-10-22 06:43:11,028 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bcb2f010c2aeaa2bad84b81d2684fa7c deactivated due to stimulus 'task-finished-1729528991.025644' +2024-10-22 06:43:11,244 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bc4eab2e4c69f1370607a11f2a9c4392 deactivated due to stimulus 'task-finished-1729528991.2393222' +2024-10-22 06:43:11,383 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 03a2135c557b0a742259d3bca3a3f5c1 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-03a2135c557b0a742259d3bca3a3f5c1' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:11,384 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 56789b3bb869078422e4e07993326c3e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-56789b3bb869078422e4e07993326c3e' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:11,491 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3c148173ebc946767fe147506184e5d5 deactivated due to stimulus 'task-finished-1729528991.4349582' +2024-10-22 06:43:11,803 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 444ea2ff016bfbcb8c2079368d20079f deactivated due to stimulus 'task-finished-1729528991.8017304' +2024-10-22 06:43:11,899 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0d7e0559a60e99a8106e3804f6ade934 deactivated due to stimulus 'task-finished-1729528991.8601727' +2024-10-22 06:43:11,935 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2482501e003e2a2f1b4ca88493771ffa initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-2482501e003e2a2f1b4ca88493771ffa' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:11,976 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc2821197bfe13c370db658898c2ecd7 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-cc2821197bfe13c370db658898c2ecd7' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:12,156 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle adf80adecdef85f30b67c30bb5c08ded initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-adf80adecdef85f30b67c30bb5c08ded' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:12,507 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 03a2135c557b0a742259d3bca3a3f5c1 deactivated due to stimulus 'task-finished-1729528992.505512' +2024-10-22 06:43:12,770 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c97865205e833b24dffa05881b7a9355 deactivated due to stimulus 'task-finished-1729528992.761427' +2024-10-22 06:43:12,812 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 554d0b89cc3e30c6daa014ddf6515849 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-554d0b89cc3e30c6daa014ddf6515849' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:13,216 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2482501e003e2a2f1b4ca88493771ffa deactivated due to stimulus 'task-finished-1729528992.9168625' +2024-10-22 06:43:13,451 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle daf954ece88a0906d0a52c3d63d0adbf initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-daf954ece88a0906d0a52c3d63d0adbf' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:13,755 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2c982b75b3cd8f58f452e3c502fc4e35 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-2c982b75b3cd8f58f452e3c502fc4e35' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:14,497 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 56789b3bb869078422e4e07993326c3e deactivated due to stimulus 'task-finished-1729528994.4278705' +2024-10-22 06:43:14,703 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle adf80adecdef85f30b67c30bb5c08ded deactivated due to stimulus 'task-finished-1729528994.6794093' +2024-10-22 06:43:14,781 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc2821197bfe13c370db658898c2ecd7 deactivated due to stimulus 'task-finished-1729528994.738729' +2024-10-22 06:43:15,038 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 519e9a5aaf5255b78a02c8a519e86848 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-519e9a5aaf5255b78a02c8a519e86848' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:15,896 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f339247e440184a6c63d166590f7ee64 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-f339247e440184a6c63d166590f7ee64' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:15,986 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c92a48712b5e61e056a3f6d130386701 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-c92a48712b5e61e056a3f6d130386701' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:16,211 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle daf954ece88a0906d0a52c3d63d0adbf deactivated due to stimulus 'task-finished-1729528996.1430075' +2024-10-22 06:43:16,378 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 30ef823fb3e4610092353d5ad026328e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-30ef823fb3e4610092353d5ad026328e' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:16,414 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 702342489700f3b202e200fe562506f8 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-702342489700f3b202e200fe562506f8' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:16,456 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 554d0b89cc3e30c6daa014ddf6515849 deactivated due to stimulus 'task-finished-1729528996.4162838' +2024-10-22 06:43:17,713 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 519e9a5aaf5255b78a02c8a519e86848 deactivated due to stimulus 'task-finished-1729528997.6774778' +2024-10-22 06:43:17,714 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2c982b75b3cd8f58f452e3c502fc4e35 deactivated due to stimulus 'task-finished-1729528997.6776395' +2024-10-22 06:43:18,051 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f339247e440184a6c63d166590f7ee64 deactivated due to stimulus 'task-finished-1729528998.0264425' +2024-10-22 06:43:18,283 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 30ef823fb3e4610092353d5ad026328e deactivated due to stimulus 'task-finished-1729528998.2666261' +2024-10-22 06:43:18,362 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ddb889047c2bc394819b94ad737c871 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-5ddb889047c2bc394819b94ad737c871' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:18,363 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cb20ca5338e4d4b923409c400a63ee36 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-cb20ca5338e4d4b923409c400a63ee36' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:18,536 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c92a48712b5e61e056a3f6d130386701 deactivated due to stimulus 'task-finished-1729528998.531684' +2024-10-22 06:43:18,795 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc3571f3c574576d181370e2b6304447 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-cc3571f3c574576d181370e2b6304447' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:43:19,390 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 52052fbfa6585c8c4a183f4f67724686 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-52052fbfa6585c8c4a183f4f67724686' executed on worker tcp://127.0.0.1:37287 +2024-10-22 06:43:19,486 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 702342489700f3b202e200fe562506f8 deactivated due to stimulus 'task-finished-1729528999.4152758' +2024-10-22 06:43:19,808 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b9d93f64b0b18e94e48adcc2627ec5dc initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-b9d93f64b0b18e94e48adcc2627ec5dc' executed on worker tcp://127.0.0.1:40781 +2024-10-22 06:44:15,716 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 56.35s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:44:15,747 - distributed.core - INFO - Event loop was unresponsive in Worker for 56.15s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:44:15,756 - distributed.core - INFO - Event loop was unresponsive in Worker for 56.13s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:44:15,799 - distributed.scheduler - INFO - Remove client Client-8264f701-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:44:15,821 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ddb889047c2bc394819b94ad737c871 deactivated due to stimulus 'remove-client-1729529055.7996662' +2024-10-22 06:44:15,823 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 52052fbfa6585c8c4a183f4f67724686 deactivated due to stimulus 'remove-client-1729529055.7996662' +2024-10-22 06:44:15,829 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc3571f3c574576d181370e2b6304447 deactivated due to stimulus 'remove-client-1729529055.7996662' +2024-10-22 06:44:15,844 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b9d93f64b0b18e94e48adcc2627ec5dc deactivated due to stimulus 'remove-client-1729529055.7996662' +2024-10-22 06:44:15,851 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cb20ca5338e4d4b923409c400a63ee36 deactivated due to stimulus 'remove-client-1729529055.7996662' +2024-10-22 06:44:15,851 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:34524; closing. +2024-10-22 06:44:15,857 - distributed.scheduler - INFO - Remove client Client-8264f701-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:44:15,868 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: 'b9d93f64b0b18e94e48adcc2627ec5dc' +2024-10-22 06:44:15,876 - distributed.scheduler - INFO - Close client connection: Client-8264f701-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:44:15,941 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:40781. Reason: worker-close +2024-10-22 06:44:15,941 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:37287. Reason: worker-close +2024-10-22 06:44:15,942 - distributed.core - ERROR - Exception while handling op shuffle_barrier +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 82, in barrier + shuffle = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: 'b9d93f64b0b18e94e48adcc2627ec5dc' +2024-10-22 06:44:15,954 - distributed.core - ERROR - Exception while handling op shuffle_receive +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: 'b9d93f64b0b18e94e48adcc2627ec5dc' +2024-10-22 06:44:15,983 - distributed.scheduler - WARNING - Attempting to reschedule task ('rechunk-p2p-8324e95b1c2ac71dc4d7463e6c44102a', 13, 1), which was not found on the scheduler. Aborting reschedule. +2024-10-22 06:44:15,983 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:44:15,983 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:44:15,984 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:44:15,996 - distributed.shuffle._comms - ERROR - 'b9d93f64b0b18e94e48adcc2627ec5dc' +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 204, in _send + return await self.rpc(address).shuffle_receive( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: 'b9d93f64b0b18e94e48adcc2627ec5dc' +2024-10-22 06:44:16,035 - distributed.comm.tcp - INFO - Connection from tcp://127.0.0.1:60428 closed before handshake completed +2024-10-22 06:44:16,047 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:34522; closing. +2024-10-22 06:44:16,048 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729529056.0485575') +2024-10-22 06:44:16,069 - distributed.comm.tcp - INFO - Connection from tcp://127.0.0.1:57200 closed before handshake completed +2024-10-22 06:44:16,070 - distributed.core - INFO - Connection to tcp://127.0.0.1:42749 has been closed. +2024-10-22 06:44:16,087 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:34508; closing. +2024-10-22 06:44:16,088 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729529056.0883484') +2024-10-22 06:44:16,088 - distributed.scheduler - INFO - Lost all workers +2024-10-22 06:44:16,093 - distributed.core - INFO - Connection to tcp://127.0.0.1:42749 has been closed. +2024-10-22 06:44:16,167 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 06:44:16,168 - distributed.scheduler - INFO - Scheduler closing all comms +__________ test_rechunk_with_partially_unknown_dimension[x5-chunks5] ___________ + +fut = , timeout = 29.663684797938913 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut +E asyncio.exceptions.CancelledError + +distributed/utils.py:1960: CancelledError + +The above exception was the direct cause of the following exception: + + async def async_fn(): + result = None + with dask.config.set(config): + async with ( + _cluster_factory() as (s, workers), + _client_factory(s) as c, + ): + args = [s] + workers + if c is not None: + args = [c] + args + try: + coro = func(*args, *outer_args, **kwargs) + task = asyncio.create_task(coro) + coro2 = utils_wait_for( + asyncio.shield(task), timeout=deadline.remaining + ) +> result = await coro2 + +distributed/utils_test.py:1011: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError + +During handling of the above exception, another exception occurred: + +fut = ._..test_func..async_fn at 0x7f220244f4d0> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1960: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1031: in async_fn + await asyncio.sleep(0.01) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +delay = 0.01, result = None + + async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result + + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: +> return await future +E asyncio.exceptions.CancelledError + +/usr/lib/python3.12/asyncio/tasks.py:665: CancelledError + +The above exception was the direct cause of the following exception: + +args = () +kwds = {'chunks': (None, (5, 5)), 'x': dask.array} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.12/contextlib.py:81: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.12/contextlib.py:81: in inner + return func(*args, **kwds) +distributed/utils_test.py:1102: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:378: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.12/asyncio/runners.py:194: in run + return runner.run(main) +/usr/lib/python3.12/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.12/asyncio/base_events.py:687: in run_until_complete + return future.result() +distributed/utils_test.py:375: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1099: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError +----------------------------- Captured stdout call ----------------------------- +Dumped cluster state to test_cluster_dump/test_rechunk_with_partially_unknown_dimension.yaml +----------------------------- Captured stderr call ----------------------------- +2024-10-22 06:44:48,788 - distributed.scheduler - INFO - State start +2024-10-22 06:44:48,806 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:38535 +2024-10-22 06:44:48,806 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45395/status +2024-10-22 06:44:48,807 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 06:44:48,851 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:45247 +2024-10-22 06:44:48,851 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:45247 +2024-10-22 06:44:48,851 - distributed.worker - INFO - Worker name: 0 +2024-10-22 06:44:48,851 - distributed.worker - INFO - dashboard at: 127.0.0.1:43549 +2024-10-22 06:44:48,852 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:38535 +2024-10-22 06:44:48,852 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:44:48,852 - distributed.worker - INFO - Threads: 1 +2024-10-22 06:44:48,852 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:44:48,852 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-a5ttnyz2 +2024-10-22 06:44:48,852 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:44:48,867 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:46675 +2024-10-22 06:44:48,867 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:46675 +2024-10-22 06:44:48,867 - distributed.worker - INFO - Worker name: 1 +2024-10-22 06:44:48,867 - distributed.worker - INFO - dashboard at: 127.0.0.1:40109 +2024-10-22 06:44:48,867 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:38535 +2024-10-22 06:44:48,867 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:44:48,867 - distributed.worker - INFO - Threads: 2 +2024-10-22 06:44:48,868 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:44:48,868 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-ew4d4akn +2024-10-22 06:44:48,868 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:44:48,947 - distributed.scheduler - INFO - Register worker +2024-10-22 06:44:48,979 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:45247 +2024-10-22 06:44:48,980 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:41298 +2024-10-22 06:44:48,993 - distributed.scheduler - INFO - Register worker +2024-10-22 06:44:49,018 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:46675 +2024-10-22 06:44:49,018 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:41310 +2024-10-22 06:44:49,019 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:44:49,020 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:44:49,029 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:38535 +2024-10-22 06:44:49,030 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:44:49,030 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:38535 +2024-10-22 06:44:49,030 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:44:49,031 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38535 +2024-10-22 06:44:49,031 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38535 +2024-10-22 06:44:49,083 - distributed.scheduler - INFO - Receive client connection: Client-c94f2c2d-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:44:49,112 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:41312 +2024-10-22 06:44:49,779 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e59664493e67b4922535dece242d1aab initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-e59664493e67b4922535dece242d1aab' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:49,780 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01e1fb53041e5fdb0d29bbe0a220a186 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-01e1fb53041e5fdb0d29bbe0a220a186' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:49,793 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 17261aa4df64de4bbec9ec8f778b6988 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-17261aa4df64de4bbec9ec8f778b6988' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:50,261 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d818f5d4a45fa1d93be4d23875883b80 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-d818f5d4a45fa1d93be4d23875883b80' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:50,263 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 31fcedb5edb6afb1da8e949ae718fd04 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-31fcedb5edb6afb1da8e949ae718fd04' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:50,287 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01db6bbc0c226a1ef7d4a6b3f462f6bb initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-01db6bbc0c226a1ef7d4a6b3f462f6bb' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:50,627 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ddb3383fd4d500ceb54cf08b5b6afae1 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-ddb3383fd4d500ceb54cf08b5b6afae1' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:50,627 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc599f874b11835571ee6bcce1edc1d8 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-cc599f874b11835571ee6bcce1edc1d8' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:50,666 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bce680412476b294e188ce6816b7e2cd initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-bce680412476b294e188ce6816b7e2cd' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:51,258 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e59664493e67b4922535dece242d1aab deactivated due to stimulus 'task-finished-1729529091.245405' +2024-10-22 06:44:51,259 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 17261aa4df64de4bbec9ec8f778b6988 deactivated due to stimulus 'task-finished-1729529091.24559' +2024-10-22 06:44:51,427 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01e1fb53041e5fdb0d29bbe0a220a186 deactivated due to stimulus 'task-finished-1729529091.4106045' +2024-10-22 06:44:51,611 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d818f5d4a45fa1d93be4d23875883b80 deactivated due to stimulus 'task-finished-1729529091.534497' +2024-10-22 06:44:51,618 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c2ee4274eae62a3e133ce55a83985341 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-c2ee4274eae62a3e133ce55a83985341' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:51,785 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 01db6bbc0c226a1ef7d4a6b3f462f6bb deactivated due to stimulus 'task-finished-1729529091.7780151' +2024-10-22 06:44:51,908 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 31fcedb5edb6afb1da8e949ae718fd04 deactivated due to stimulus 'task-finished-1729529091.9021618' +2024-10-22 06:44:52,258 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e765252ecfc2b6ffd5668c3b385a3174 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-e765252ecfc2b6ffd5668c3b385a3174' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:52,259 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bce680412476b294e188ce6816b7e2cd deactivated due to stimulus 'task-finished-1729529092.2337945' +2024-10-22 06:44:52,263 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 84386ba0af6f45a3973ad458a14d56c5 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-84386ba0af6f45a3973ad458a14d56c5' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:52,484 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ddb3383fd4d500ceb54cf08b5b6afae1 deactivated due to stimulus 'task-finished-1729529092.4524372' +2024-10-22 06:44:52,637 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c2ee4274eae62a3e133ce55a83985341 deactivated due to stimulus 'task-finished-1729529092.5827184' +2024-10-22 06:44:52,640 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c81c046e65c84ae49e04fd567198f65c initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-c81c046e65c84ae49e04fd567198f65c' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:52,671 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d739332393d2d6dae0f074f82be858b3 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-d739332393d2d6dae0f074f82be858b3' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:52,799 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc599f874b11835571ee6bcce1edc1d8 deactivated due to stimulus 'task-finished-1729529092.7415996' +2024-10-22 06:44:52,933 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 59881a6059b5a68660aef4b1433f259a initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-59881a6059b5a68660aef4b1433f259a' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:53,356 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec3af6e3853fbf1e077a45ec7a358cac initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-ec3af6e3853fbf1e077a45ec7a358cac' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:53,486 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e765252ecfc2b6ffd5668c3b385a3174 deactivated due to stimulus 'task-finished-1729529093.4687536' +2024-10-22 06:44:53,638 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e67f678006c2dfb420382b0653e7e278 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-e67f678006c2dfb420382b0653e7e278' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:54,142 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec792f9b679daef65c0ec6556b5e9c9f initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-ec792f9b679daef65c0ec6556b5e9c9f' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:54,239 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c81c046e65c84ae49e04fd567198f65c deactivated due to stimulus 'task-finished-1729529094.2016137' +2024-10-22 06:44:54,780 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 59881a6059b5a68660aef4b1433f259a deactivated due to stimulus 'task-finished-1729529094.7429855' +2024-10-22 06:44:54,807 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b79d0d8845743010f9e3088a0beb18d1 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-b79d0d8845743010f9e3088a0beb18d1' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:55,074 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 84386ba0af6f45a3973ad458a14d56c5 deactivated due to stimulus 'task-finished-1729529094.9715421' +2024-10-22 06:44:55,416 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f9160caf490b8329e7669ecbd1ac041e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-f9160caf490b8329e7669ecbd1ac041e' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:56,001 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e67f678006c2dfb420382b0653e7e278 deactivated due to stimulus 'task-finished-1729529095.9538443' +2024-10-22 06:44:56,034 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle df475960043b4dab9b72a8cebe24fbf0 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-df475960043b4dab9b72a8cebe24fbf0' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:56,187 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e77aa0a14aa79e0a383a0b353abba873 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-e77aa0a14aa79e0a383a0b353abba873' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:56,261 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d739332393d2d6dae0f074f82be858b3 deactivated due to stimulus 'task-finished-1729529096.1629183' +2024-10-22 06:44:56,484 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec3af6e3853fbf1e077a45ec7a358cac deactivated due to stimulus 'task-finished-1729529096.4629312' +2024-10-22 06:44:56,758 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle ec792f9b679daef65c0ec6556b5e9c9f deactivated due to stimulus 'task-finished-1729529096.7336948' +2024-10-22 06:44:56,970 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 714ef9e25e8b5c1d5b3451d982311c2e initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-714ef9e25e8b5c1d5b3451d982311c2e' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:57,674 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b8fcb44d156414c630e1d6ff0df77c1b initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-b8fcb44d156414c630e1d6ff0df77c1b' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:57,690 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b79d0d8845743010f9e3088a0beb18d1 deactivated due to stimulus 'task-finished-1729529097.5403302' +2024-10-22 06:44:58,018 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b9c215f5d25889bd6c1d4a0fb35eb468 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-b9c215f5d25889bd6c1d4a0fb35eb468' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:58,049 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle df475960043b4dab9b72a8cebe24fbf0 deactivated due to stimulus 'task-finished-1729529097.9584105' +2024-10-22 06:44:58,248 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 57d52a42fcd6be719e65e6f9840d0cdb initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-57d52a42fcd6be719e65e6f9840d0cdb' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:44:58,523 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f385202a9df8dca26b51e3b27030056b initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-f385202a9df8dca26b51e3b27030056b' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:59,249 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 11f1f6ff4fd0902dbfb6a01bfdb5359d initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-11f1f6ff4fd0902dbfb6a01bfdb5359d' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:44:59,763 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 714ef9e25e8b5c1d5b3451d982311c2e deactivated due to stimulus 'task-finished-1729529099.747171' +2024-10-22 06:44:59,929 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f9160caf490b8329e7669ecbd1ac041e deactivated due to stimulus 'task-finished-1729529099.8982058' +2024-10-22 06:45:00,260 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f385202a9df8dca26b51e3b27030056b deactivated due to stimulus 'task-finished-1729529100.2422922' +2024-10-22 06:45:00,410 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 57d52a42fcd6be719e65e6f9840d0cdb deactivated due to stimulus 'task-finished-1729529100.3423104' +2024-10-22 06:45:00,525 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cf4000d0bbe7b2713a9b81d90981fe9b initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-cf4000d0bbe7b2713a9b81d90981fe9b' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:00,599 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 1164a44fb53f1d99c26d483a9f7618c2 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-1164a44fb53f1d99c26d483a9f7618c2' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:00,615 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle e77aa0a14aa79e0a383a0b353abba873 deactivated due to stimulus 'task-finished-1729529100.5282543' +2024-10-22 06:45:01,103 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b8fcb44d156414c630e1d6ff0df77c1b deactivated due to stimulus 'task-finished-1729529101.0132995' +2024-10-22 06:45:01,221 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3eabd736c25cbfb05d814f9f3d9bfc98 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-3eabd736c25cbfb05d814f9f3d9bfc98' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:01,270 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle b9c215f5d25889bd6c1d4a0fb35eb468 deactivated due to stimulus 'task-finished-1729529101.223212' +2024-10-22 06:45:01,793 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 11f1f6ff4fd0902dbfb6a01bfdb5359d deactivated due to stimulus 'task-finished-1729529101.765424' +2024-10-22 06:45:01,871 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0fa082cb36216f52a7836b89276b16a9 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-0fa082cb36216f52a7836b89276b16a9' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:02,516 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 4a20b1bca8a7c0bc5fe798d5d474efe7 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-4a20b1bca8a7c0bc5fe798d5d474efe7' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:02,816 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 8f6377159296c2aea721b80be7444955 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-8f6377159296c2aea721b80be7444955' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:02,955 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cf4000d0bbe7b2713a9b81d90981fe9b deactivated due to stimulus 'task-finished-1729529102.81532' +2024-10-22 06:45:03,449 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3eabd736c25cbfb05d814f9f3d9bfc98 deactivated due to stimulus 'task-finished-1729529103.4023967' +2024-10-22 06:45:03,652 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 23c3e0a6318f8719413afcb7fb1c165c initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-23c3e0a6318f8719413afcb7fb1c165c' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:03,898 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 1164a44fb53f1d99c26d483a9f7618c2 deactivated due to stimulus 'task-finished-1729529103.768584' +2024-10-22 06:45:03,951 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2037175d04234fb783dd45ef54a900ac initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-2037175d04234fb783dd45ef54a900ac' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:05,234 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 4a20b1bca8a7c0bc5fe798d5d474efe7 deactivated due to stimulus 'task-finished-1729529105.2185748' +2024-10-22 06:45:05,504 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 8f6377159296c2aea721b80be7444955 deactivated due to stimulus 'task-finished-1729529105.4625776' +2024-10-22 06:45:05,801 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0cc7ac64edd3e809e1ee98f4e84d65e4 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-0cc7ac64edd3e809e1ee98f4e84d65e4' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:05,914 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 23c3e0a6318f8719413afcb7fb1c165c deactivated due to stimulus 'task-finished-1729529105.8030348' +2024-10-22 06:45:06,240 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0fa082cb36216f52a7836b89276b16a9 deactivated due to stimulus 'task-finished-1729529105.964419' +2024-10-22 06:45:06,262 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 92126407b7617e8e1154192b00785cb2 initialized by task 'concatenate-from-dask-array-rechunk-slice-rechunk-transfer-values-shuffle-barrier-92126407b7617e8e1154192b00785cb2' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:06,637 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 204b3c4c3770c441622961008d21fe35 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-204b3c4c3770c441622961008d21fe35' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:06,782 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle fd6cd470d74c25edf76b3f4dc74c12c0 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-fd6cd470d74c25edf76b3f4dc74c12c0' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:07,000 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2037175d04234fb783dd45ef54a900ac deactivated due to stimulus 'task-finished-1729529106.9974322' +2024-10-22 06:45:07,241 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0cc7ac64edd3e809e1ee98f4e84d65e4 deactivated due to stimulus 'task-finished-1729529107.2223475' +2024-10-22 06:45:07,299 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 92126407b7617e8e1154192b00785cb2 deactivated due to stimulus 'task-finished-1729529107.2972493' +2024-10-22 06:45:07,436 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 204b3c4c3770c441622961008d21fe35 deactivated due to stimulus 'task-finished-1729529107.4297621' +2024-10-22 06:45:07,455 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle fd6cd470d74c25edf76b3f4dc74c12c0 deactivated due to stimulus 'task-finished-1729529107.4534297' +2024-10-22 06:45:08,257 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d07954b281dba89624cbc6c762797e1f initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-d07954b281dba89624cbc6c762797e1f' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:08,291 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 38071ec7b016834cb180426b1b3240a7 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-38071ec7b016834cb180426b1b3240a7' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:08,419 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 806d3b72ee98acf548c7fd2bbb441c36 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-806d3b72ee98acf548c7fd2bbb441c36' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:08,966 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bcb2f010c2aeaa2bad84b81d2684fa7c initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-bcb2f010c2aeaa2bad84b81d2684fa7c' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:09,210 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle aa36ff673eca80feec08ff85465e098e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-aa36ff673eca80feec08ff85465e098e' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:09,273 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3c148173ebc946767fe147506184e5d5 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-3c148173ebc946767fe147506184e5d5' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:11,421 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bc4eab2e4c69f1370607a11f2a9c4392 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-bc4eab2e4c69f1370607a11f2a9c4392' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:11,822 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 444ea2ff016bfbcb8c2079368d20079f initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-444ea2ff016bfbcb8c2079368d20079f' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:12,635 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle d07954b281dba89624cbc6c762797e1f deactivated due to stimulus 'task-finished-1729529112.5942504' +2024-10-22 06:45:12,760 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 38071ec7b016834cb180426b1b3240a7 deactivated due to stimulus 'task-finished-1729529112.7574859' +2024-10-22 06:45:12,875 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c97865205e833b24dffa05881b7a9355 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-c97865205e833b24dffa05881b7a9355' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:12,956 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 806d3b72ee98acf548c7fd2bbb441c36 deactivated due to stimulus 'task-finished-1729529112.921604' +2024-10-22 06:45:13,587 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle aa36ff673eca80feec08ff85465e098e deactivated due to stimulus 'task-finished-1729529113.5624628' +2024-10-22 06:45:13,750 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 03a2135c557b0a742259d3bca3a3f5c1 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-03a2135c557b0a742259d3bca3a3f5c1' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:13,920 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bcb2f010c2aeaa2bad84b81d2684fa7c deactivated due to stimulus 'task-finished-1729529113.8923714' +2024-10-22 06:45:14,274 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 56789b3bb869078422e4e07993326c3e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-56789b3bb869078422e4e07993326c3e' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:14,339 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 3c148173ebc946767fe147506184e5d5 deactivated due to stimulus 'task-finished-1729529114.2459788' +2024-10-22 06:45:15,103 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle bc4eab2e4c69f1370607a11f2a9c4392 deactivated due to stimulus 'task-finished-1729529115.0814767' +2024-10-22 06:45:15,270 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 03a2135c557b0a742259d3bca3a3f5c1 deactivated due to stimulus 'task-finished-1729529115.2598567' +2024-10-22 06:45:15,292 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2482501e003e2a2f1b4ca88493771ffa initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-2482501e003e2a2f1b4ca88493771ffa' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:15,406 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle adf80adecdef85f30b67c30bb5c08ded initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-adf80adecdef85f30b67c30bb5c08ded' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:15,439 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 444ea2ff016bfbcb8c2079368d20079f deactivated due to stimulus 'task-finished-1729529115.429617' +2024-10-22 06:45:15,503 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0d7e0559a60e99a8106e3804f6ade934 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-0d7e0559a60e99a8106e3804f6ade934' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:15,532 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc2821197bfe13c370db658898c2ecd7 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-cc2821197bfe13c370db658898c2ecd7' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:15,761 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 554d0b89cc3e30c6daa014ddf6515849 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-554d0b89cc3e30c6daa014ddf6515849' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:16,039 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c97865205e833b24dffa05881b7a9355 deactivated due to stimulus 'task-finished-1729529116.0022666' +2024-10-22 06:45:16,376 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2482501e003e2a2f1b4ca88493771ffa deactivated due to stimulus 'task-finished-1729529116.3733304' +2024-10-22 06:45:16,659 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle adf80adecdef85f30b67c30bb5c08ded deactivated due to stimulus 'task-finished-1729529116.6575682' +2024-10-22 06:45:16,788 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle daf954ece88a0906d0a52c3d63d0adbf initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-daf954ece88a0906d0a52c3d63d0adbf' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:16,989 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 0d7e0559a60e99a8106e3804f6ade934 deactivated due to stimulus 'task-finished-1729529116.899771' +2024-10-22 06:45:16,992 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 519e9a5aaf5255b78a02c8a519e86848 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-519e9a5aaf5255b78a02c8a519e86848' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:17,116 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 56789b3bb869078422e4e07993326c3e deactivated due to stimulus 'task-finished-1729529117.0244699' +2024-10-22 06:45:17,279 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2c982b75b3cd8f58f452e3c502fc4e35 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-2c982b75b3cd8f58f452e3c502fc4e35' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:17,327 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc2821197bfe13c370db658898c2ecd7 deactivated due to stimulus 'task-finished-1729529117.303363' +2024-10-22 06:45:17,504 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c92a48712b5e61e056a3f6d130386701 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-c92a48712b5e61e056a3f6d130386701' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:17,648 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 554d0b89cc3e30c6daa014ddf6515849 deactivated due to stimulus 'task-finished-1729529117.544368' +2024-10-22 06:45:17,749 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f339247e440184a6c63d166590f7ee64 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-f339247e440184a6c63d166590f7ee64' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:17,988 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 30ef823fb3e4610092353d5ad026328e initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-30ef823fb3e4610092353d5ad026328e' executed on worker tcp://127.0.0.1:45247 +2024-10-22 06:45:18,330 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc3571f3c574576d181370e2b6304447 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-cc3571f3c574576d181370e2b6304447' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:45:18,494 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle daf954ece88a0906d0a52c3d63d0adbf deactivated due to stimulus 'task-finished-1729529118.450363' +2024-10-22 06:45:18,699 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 519e9a5aaf5255b78a02c8a519e86848 deactivated due to stimulus 'task-finished-1729529118.6648827' +2024-10-22 06:46:07,026 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 48.70s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:46:07,047 - distributed.core - INFO - Event loop was unresponsive in Worker for 48.49s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:46:07,068 - distributed.core - INFO - Event loop was unresponsive in Worker for 48.48s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2024-10-22 06:46:07,159 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ddb889047c2bc394819b94ad737c871 initialized by task 'concatenate-rechunk-transfer-shuffle-barrier-5ddb889047c2bc394819b94ad737c871' executed on worker tcp://127.0.0.1:46675 +2024-10-22 06:46:07,207 - distributed.scheduler - INFO - Remove client Client-c94f2c2d-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:46:07,218 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle f339247e440184a6c63d166590f7ee64 deactivated due to stimulus 'remove-client-1729529167.2076478' +2024-10-22 06:46:07,219 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 30ef823fb3e4610092353d5ad026328e deactivated due to stimulus 'remove-client-1729529167.2076478' +2024-10-22 06:46:07,220 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 2c982b75b3cd8f58f452e3c502fc4e35 deactivated due to stimulus 'remove-client-1729529167.2076478' +2024-10-22 06:46:07,241 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle cc3571f3c574576d181370e2b6304447 deactivated due to stimulus 'remove-client-1729529167.2076478' +2024-10-22 06:46:07,242 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle 5ddb889047c2bc394819b94ad737c871 deactivated due to stimulus 'remove-client-1729529167.2076478' +2024-10-22 06:46:07,244 - distributed.shuffle._scheduler_plugin - WARNING - Shuffle c92a48712b5e61e056a3f6d130386701 deactivated due to stimulus 'remove-client-1729529167.2076478' +2024-10-22 06:46:07,260 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:41312; closing. +2024-10-22 06:46:07,273 - distributed.scheduler - INFO - Remove client Client-c94f2c2d-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:46:07,299 - distributed.scheduler - INFO - Close client connection: Client-c94f2c2d-8fcb-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:46:07,299 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:45247. Reason: worker-close +2024-10-22 06:46:07,299 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:46675. Reason: worker-close +2024-10-22 06:46:07,397 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:46:07,397 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:46:07,397 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 06:46:07,452 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,468 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,483 - distributed.core - ERROR - Exception while handling op shuffle_get +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ~~~~~~~~~~~~~~~~~~~~^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,504 - distributed.core - ERROR - Exception while handling op shuffle_receive +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,555 - distributed.core - ERROR - Exception while handling op shuffle_receive +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,622 - distributed.shuffle._comms - ERROR - '5ddb889047c2bc394819b94ad737c871' +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 204, in _send + return await self.rpc(address).shuffle_receive( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,671 - distributed.shuffle._comms - ERROR - '5ddb889047c2bc394819b94ad737c871' +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 836, in wrapper + return await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_comms.py", line 72, in _process + await self.send(address, shards) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 226, in send + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 421, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_core.py", line 204, in _send + return await self.rpc(address).shuffle_receive( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 970, in _handle_comm + result = await result + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 314, in shuffle_receive + shuffle_run = await self._get_shuffle_run(shuffle_id, run_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 369, in _get_shuffle_run + return await self.shuffle_runs.get_with_run_id( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 118, in get_with_run_id + shuffle_run = await self._refresh(shuffle_id=shuffle_id) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 225, in _refresh + result = await self._fetch(shuffle_id=shuffle_id, spec=spec, key=key) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_worker_plugin.py", line 189, in _fetch + result = await self._plugin.worker.scheduler.shuffle_get( + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1395, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1179, in send_recv + raise exc.with_traceback(tb) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 968, in _handle_comm + result = handler(**msg) + ^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/shuffle/_scheduler_plugin.py", line 128, in get + state = self.active_shuffles[id] + ^^^^^^^^^^^^^^^^^ +KeyError: '5ddb889047c2bc394819b94ad737c871' +2024-10-22 06:46:07,695 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:41298; closing. +2024-10-22 06:46:07,696 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729529167.6961637') +2024-10-22 06:46:07,696 - distributed.core - INFO - Connection to tcp://127.0.0.1:38535 has been closed. +2024-10-22 06:46:07,747 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:41310; closing. +2024-10-22 06:46:07,748 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729529167.748279') +2024-10-22 06:46:07,748 - distributed.scheduler - INFO - Lost all workers +2024-10-22 06:46:07,748 - distributed.core - INFO - Connection to tcp://127.0.0.1:38535 has been closed. +2024-10-22 06:46:07,790 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 06:46:07,791 - distributed.scheduler - INFO - Scheduler closing all comms +_ test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] _ + +fut = , timeout = 0.5 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1960: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = Scheduler local=tcp://127.0.0.1:54298 remote=tcp://127.0.0.1:37317> +deserializers = None + + async def read(self, deserializers=None): + stream = self.stream + if stream is None: + raise CommClosedError() + + fmt = "Q" + fmt_size = struct.calcsize(fmt) + + try: + # Don't store multiple numpy or parquet buffers into the same buffer, or + # none will be released until all are released. +> frames_nosplit_nbytes_bin = await stream.read_bytes(fmt_size) +E asyncio.exceptions.CancelledError + +distributed/comm/tcp.py:225: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {'close_worker': True, 'intermediate_state': 'cancelled'} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.12/contextlib.py:81: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.12/contextlib.py:81: in inner + return func(*args, **kwds) +distributed/utils_test.py:1102: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:378: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.12/asyncio/runners.py:194: in run + return runner.run(main) +/usr/lib/python3.12/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.12/asyncio/base_events.py:687: in run_until_complete + return future.result() +distributed/utils_test.py:375: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1099: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1960: in wait_for + return await fut +distributed/utils_test.py:1000: in async_fn + _client_factory(s) as c, +/usr/lib/python3.12/contextlib.py:210: in __aenter__ + return await anext(self.gen) +distributed/utils_test.py:949: in _client_factory + async with Client( +distributed/client.py:1481: in __aenter__ + await self +distributed/client.py:1299: in _start + await self._ensure_connected(timeout=timeout) +distributed/client.py:1385: in _ensure_connected + msg = await wait_for(comm.read(), timeout) +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError +----------------------------- Captured stderr call ----------------------------- +2024-10-22 06:51:56,776 - distributed.scheduler - INFO - State start +2024-10-22 06:51:56,781 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:37317 +2024-10-22 06:51:56,781 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45729/status +2024-10-22 06:51:56,781 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 06:51:56,798 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:45635 +2024-10-22 06:51:56,798 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:45635 +2024-10-22 06:51:56,799 - distributed.worker - INFO - Worker name: 0 +2024-10-22 06:51:56,799 - distributed.worker - INFO - dashboard at: 127.0.0.1:43479 +2024-10-22 06:51:56,799 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:37317 +2024-10-22 06:51:56,799 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:51:56,799 - distributed.worker - INFO - Threads: 1 +2024-10-22 06:51:56,799 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:51:56,799 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-zanm9xx6 +2024-10-22 06:51:56,799 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:51:56,813 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:46199 +2024-10-22 06:51:56,814 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:46199 +2024-10-22 06:51:56,814 - distributed.worker - INFO - Worker name: 1 +2024-10-22 06:51:56,814 - distributed.worker - INFO - dashboard at: 127.0.0.1:45065 +2024-10-22 06:51:56,814 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:37317 +2024-10-22 06:51:56,814 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:51:56,814 - distributed.worker - INFO - Threads: 2 +2024-10-22 06:51:56,814 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 06:51:56,814 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-lniag47t +2024-10-22 06:51:56,814 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:51:56,856 - distributed.scheduler - INFO - Register worker +2024-10-22 06:51:56,883 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:45635 +2024-10-22 06:51:56,901 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54282 +2024-10-22 06:51:56,902 - distributed.scheduler - INFO - Register worker +2024-10-22 06:51:56,949 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:46199 +2024-10-22 06:51:56,993 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54284 +2024-10-22 06:51:56,994 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:51:56,994 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 06:51:56,995 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:37317 +2024-10-22 06:51:56,995 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:51:56,995 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:37317 +2024-10-22 06:51:56,995 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 06:51:56,996 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:37317 +2024-10-22 06:51:56,996 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:37317 +2024-10-22 06:51:57,076 - distributed.scheduler - INFO - Receive client connection: Client-c8657398-8fcc-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:51:57,275 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54298 +2024-10-22 06:51:57,599 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:45635. Reason: worker-close +2024-10-22 06:51:57,673 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:46199. Reason: worker-close +2024-10-22 06:51:57,673 - distributed.core - INFO - Connection to tcp://127.0.0.1:54298 has been closed. +2024-10-22 06:51:57,673 - distributed.scheduler - INFO - Remove client Client-c8657398-8fcc-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:51:57,676 - distributed.scheduler - INFO - Close client connection: Client-c8657398-8fcc-11ef-ad33-7d0752ea3fc4 +2024-10-22 06:51:57,682 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:54282; closing. +2024-10-22 06:51:57,682 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:54284; closing. +2024-10-22 06:51:57,683 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729529517.6832204') +2024-10-22 06:51:57,683 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729529517.683667') +2024-10-22 06:51:57,683 - distributed.scheduler - INFO - Lost all workers +2024-10-22 06:51:57,684 - distributed.core - INFO - Connection to tcp://127.0.0.1:37317 has been closed. +2024-10-22 06:51:57,697 - distributed.core - INFO - Connection to tcp://127.0.0.1:37317 has been closed. +2024-10-22 06:51:58,098 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 06:51:58,099 - distributed.scheduler - INFO - Scheduler closing all comms +______________________________ test_run_rpc_error ______________________________ + +fut = , timeout = 0.2 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1960: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = Scheduler local=tcp://127.0.0.1:38324 remote=tcp://127.0.0.1:39985> +deserializers = None + + async def read(self, deserializers=None): + stream = self.stream + if stream is None: + raise CommClosedError() + + fmt = "Q" + fmt_size = struct.calcsize(fmt) + + try: + # Don't store multiple numpy or parquet buffers into the same buffer, or + # none will be released until all are released. +> frames_nosplit_nbytes_bin = await stream.read_bytes(fmt_size) +E asyncio.exceptions.CancelledError + +distributed/comm/tcp.py:225: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.12/contextlib.py:81: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.12/contextlib.py:81: in inner + return func(*args, **kwds) +distributed/utils_test.py:1102: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:378: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.12/asyncio/runners.py:194: in run + return runner.run(main) +/usr/lib/python3.12/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.12/asyncio/base_events.py:687: in run_until_complete + return future.result() +distributed/utils_test.py:375: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1099: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1960: in wait_for + return await fut +distributed/utils_test.py:1000: in async_fn + _client_factory(s) as c, +/usr/lib/python3.12/contextlib.py:210: in __aenter__ + return await anext(self.gen) +distributed/utils_test.py:949: in _client_factory + async with Client( +distributed/client.py:1481: in __aenter__ + await self +distributed/client.py:1299: in _start + await self._ensure_connected(timeout=timeout) +distributed/client.py:1385: in _ensure_connected + msg = await wait_for(comm.read(), timeout) +distributed/utils.py:1959: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + # Since there are no new cancel requests, we're + # handling this. +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.12/asyncio/timeouts.py:115: TimeoutError +----------------------------- Captured stderr call ----------------------------- +2024-10-22 07:01:03,663 - distributed.scheduler - INFO - State start +2024-10-22 07:01:03,799 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:39985 +2024-10-22 07:01:03,800 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:41567/status +2024-10-22 07:01:03,800 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 07:01:04,233 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:37699 +2024-10-22 07:01:04,234 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:37699 +2024-10-22 07:01:04,234 - distributed.worker - INFO - Worker name: 0 +2024-10-22 07:01:04,234 - distributed.worker - INFO - dashboard at: 127.0.0.1:35619 +2024-10-22 07:01:04,234 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:39985 +2024-10-22 07:01:04,234 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:01:04,234 - distributed.worker - INFO - Threads: 1 +2024-10-22 07:01:04,234 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 07:01:04,234 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-11c43ll0 +2024-10-22 07:01:04,235 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:01:04,236 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:46199 +2024-10-22 07:01:04,236 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:46199 +2024-10-22 07:01:04,236 - distributed.worker - INFO - Worker name: 1 +2024-10-22 07:01:04,236 - distributed.worker - INFO - dashboard at: 127.0.0.1:38787 +2024-10-22 07:01:04,236 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:39985 +2024-10-22 07:01:04,236 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:01:04,236 - distributed.worker - INFO - Threads: 2 +2024-10-22 07:01:04,236 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 07:01:04,236 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-xlzuz_me +2024-10-22 07:01:04,485 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:01:04,550 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:39985 +2024-10-22 07:01:04,550 - distributed.comm.tcp - INFO - Connection from tcp://127.0.0.1:38284 closed before handshake completed +2024-10-22 07:01:04,551 - distributed.scheduler - INFO - Register worker +2024-10-22 07:01:04,567 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:46199 +2024-10-22 07:01:04,567 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38298 +2024-10-22 07:01:04,568 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 07:01:04,568 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:39985 +2024-10-22 07:01:04,568 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:01:04,569 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:39985 +2024-10-22 07:01:04,723 - distributed.scheduler - INFO - Register worker +2024-10-22 07:01:04,958 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:37699 +2024-10-22 07:01:05,157 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38312 +2024-10-22 07:01:05,160 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 07:01:05,170 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:39985 +2024-10-22 07:01:05,170 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:01:05,171 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:39985 +2024-10-22 07:01:05,244 - distributed.scheduler - INFO - Receive client connection: Client-0f2242d6-8fce-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:01:05,438 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38324 +2024-10-22 07:01:05,560 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:37699. Reason: worker-close +2024-10-22 07:01:05,560 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:46199. Reason: worker-close +2024-10-22 07:01:05,569 - distributed.core - INFO - Connection to tcp://127.0.0.1:38324 has been closed. +2024-10-22 07:01:05,721 - distributed.scheduler - INFO - Remove client Client-0f2242d6-8fce-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:01:05,736 - distributed.scheduler - INFO - Close client connection: Client-0f2242d6-8fce-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:01:05,819 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat. +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1620, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1564, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils.py", line 1960, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1611, in connect + async with asyncio.timeout(math.inf) as scope: + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/worker.py", line 1252, in heartbeat + response = await retry_operation( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 452, in retry_operation + return await retry( + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/utils_comm.py", line 431, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1392, in send_recv_from_rpc + comm = await self.pool.connect(self.addr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/core.py", line 1632, in connect + raise CommClosedError(reason) +distributed.comm.core.CommClosedError: ConnectionPool closing. +2024-10-22 07:01:05,850 - distributed.comm.tcp - INFO - Connection from tcp://127.0.0.1:38344 closed before handshake completed +2024-10-22 07:01:05,851 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:38312; closing. +2024-10-22 07:01:05,851 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729530065.8513575') +2024-10-22 07:01:05,851 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:38298; closing. +2024-10-22 07:01:05,851 - distributed.core - INFO - Connection to tcp://127.0.0.1:39985 has been closed. +2024-10-22 07:01:05,866 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729530065.866048') +2024-10-22 07:01:05,866 - distributed.scheduler - INFO - Lost all workers +2024-10-22 07:01:05,866 - distributed.core - INFO - Connection to tcp://127.0.0.1:39985 has been closed. +2024-10-22 07:01:05,885 - distributed.batched - INFO - Batched Comm Closed +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/batched.py", line 115, in _background_send + nbytes = yield coro + ^^^^^^^^^^ + File "/usr/lib/python3/dist-packages/tornado/gen.py", line 766, in run + value = future.result() + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build/distributed/comm/tcp.py", line 262, in write + raise CommClosedError() +distributed.comm.core.CommClosedError +2024-10-22 07:01:05,910 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 07:01:05,910 - distributed.scheduler - INFO - Scheduler closing all comms +___________________ test_last_in_first_out[queue on worker] ____________________ + +c = +s = +a = +pause = False + + @gen_blockable_cluster + async def test_last_in_first_out(c, s, a, pause): + async with block_worker(c, s, a, pause, 15, 5): + xs = [c.submit(slowinc, i, delay=0.05, key=f"x{i}") for i in range(5)] + ys = [c.submit(slowinc, xs[i], delay=0.05, key=f"y{i}") for i in range(5)] + zs = [c.submit(slowinc, ys[i], delay=0.05, key=f"z{i}") for i in range(5)] + + while not any(s.tasks[z.key].state == "memory" for z in zs): + await asyncio.sleep(0.01) +> assert not all(s.tasks[x.key].state == "memory" for x in xs) +E assert not True +E + where True = all(. at 0x7f22015492f0>) + +distributed/tests/test_priorities.py:234: AssertionError +----------------------------- Captured stdout call ----------------------------- +Dumped cluster state to test_cluster_dump/test_last_in_first_out.yaml +----------------------------- Captured stderr call ----------------------------- +2024-10-22 07:26:16,514 - distributed.scheduler - INFO - State start +2024-10-22 07:26:16,598 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:38475 +2024-10-22 07:26:16,920 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42317/status +2024-10-22 07:26:16,920 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 07:26:17,011 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:45933 +2024-10-22 07:26:17,011 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:45933 +2024-10-22 07:26:17,011 - distributed.worker - INFO - Worker name: 0 +2024-10-22 07:26:17,012 - distributed.worker - INFO - dashboard at: 127.0.0.1:40203 +2024-10-22 07:26:17,012 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:38475 +2024-10-22 07:26:17,012 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:26:17,012 - distributed.worker - INFO - Threads: 1 +2024-10-22 07:26:17,012 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 07:26:17,012 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-bcnblj4a +2024-10-22 07:26:17,012 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:26:17,123 - distributed.scheduler - INFO - Register worker +2024-10-22 07:26:17,195 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:45933 +2024-10-22 07:26:17,389 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:48726 +2024-10-22 07:26:17,390 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 07:26:17,391 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:38475 +2024-10-22 07:26:17,391 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:26:17,401 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38475 +2024-10-22 07:26:17,490 - distributed.scheduler - INFO - Receive client connection: Client-947e5168-8fd1-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:26:17,590 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:48732 +2024-10-22 07:26:20,371 - distributed.scheduler - INFO - Remove client Client-947e5168-8fd1-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:26:20,401 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:48732; closing. +2024-10-22 07:26:20,402 - distributed.scheduler - INFO - Remove client Client-947e5168-8fd1-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:26:20,419 - distributed.scheduler - INFO - Close client connection: Client-947e5168-8fd1-11ef-ad33-7d0752ea3fc4 +2024-10-22 07:26:20,446 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:45933. Reason: worker-close +2024-10-22 07:26:20,463 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 07:26:20,483 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:48726; closing. +2024-10-22 07:26:20,491 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729531580.4909594') +2024-10-22 07:26:20,491 - distributed.scheduler - INFO - Lost all workers +2024-10-22 07:26:20,491 - distributed.core - INFO - Connection to tcp://127.0.0.1:38475 has been closed. +2024-10-22 07:26:20,506 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 07:26:20,506 - distributed.scheduler - INFO - Scheduler closing all comms ============================= slowest 20 durations ============================= -10.02s call distributed/tests/test_utils_test.py::test_popen_timeout -9.05s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_method -8.09s call distributed/tests/test_nanny.py::test_num_fds -6.34s call distributed/tests/test_failed_workers.py::test_worker_doesnt_await_task_completion -5.40s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x4-chunks4] -5.26s call distributed/tests/test_failed_workers.py::test_forgotten_futures_dont_clean_up_new_futures -5.22s call distributed/tests/test_stress.py::test_cancel_stress_sync -5.14s call distributed/tests/test_steal.py::test_balance_with_longer_task -5.02s call distributed/tests/test_stress.py::test_cancel_stress -4.91s call distributed/tests/test_tls_functional.py::test_retire_workers -4.90s call distributed/tests/test_chaos.py::test_KillWorker[sys.exit] -4.52s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] -4.38s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] -4.28s call distributed/diagnostics/tests/test_progress.py::test_group_timing -3.80s call distributed/tests/test_failed_workers.py::test_restart_sync -3.74s call distributed/tests/test_failed_workers.py::test_restart_during_computation -3.71s call distributed/tests/test_failed_workers.py::test_multiple_clients_restart -3.51s call distributed/tests/test_nanny.py::test_nanny_process_failure -3.50s call distributed/tests/test_nanny.py::test_environ_plugin -3.49s call distributed/tests/test_utils_test.py::test_bare_cluster +189.59s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_method +86.42s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] +79.05s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] +30.15s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x4-chunks4] +27.10s call distributed/tests/test_nanny.py::test_environ_plugin +24.21s call distributed/tests/test_nanny.py::test_num_fds +22.93s call distributed/tests/test_scheduler.py::test_include_communication_in_occupancy +22.32s call distributed/tests/test_nanny.py::test_nanny_plugin_register_during_start_success[True] +21.95s call distributed/tests/test_failed_workers.py::test_restart_sync +21.28s call distributed/diagnostics/tests/test_progress.py::test_group_timing +20.19s call distributed/protocol/tests/test_serialize.py::test_large_pickled_object +19.08s call distributed/tests/test_failed_workers.py::test_restart_during_computation +18.74s call distributed/tests/test_failed_workers.py::test_restart_scheduler +18.65s call distributed/diagnostics/tests/test_progress.py::test_AllProgress +17.45s call distributed/tests/test_nanny.py::test_nanny_plugin_simple +17.37s call distributed/tests/test_client.py::test_repr[func2] +16.15s setup distributed/tests/test_publish.py::test_datasets_getitem_default +15.82s setup distributed/tests/test_publish.py::test_unpublish_multiple_datasets_sync +15.72s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_fully_unknown_dimension[x3-chunks3] +15.55s call distributed/tests/test_failed_workers.py::test_multiple_clients_restart =========================== short test summary info ============================ SKIPPED [1] distributed/cli/tests/test_dask_ssh.py:9: could not import 'paramiko': No module named 'paramiko' SKIPPED [1] distributed/comm/tests/test_ucx.py:15: could not import 'ucp': No module named 'ucp' @@ -6817,8 +8895,225 @@ SKIPPED [1] distributed/tests/test_worker_memory.py:871: need --runslow option to run SKIPPED [2] distributed/tests/test_worker_memory.py:883: need --runslow option to run SKIPPED [1] distributed/tests/test_worker_memory.py:997: need --runslow option to run -= 2865 passed, 270 skipped, 221 deselected, 15 xfailed, 8 xpassed in 1018.66s (0:16:58) = -*** END OF RUN 1: ALL TESTS RUN HAVE NOW PASSED/XFAILED *** +FAILED distributed/shuffle/tests/test_rechunk.py::test_rechunk_method - TimeoutError +FAILED distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] - TimeoutError +FAILED distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] - TimeoutError +FAILED distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] - TimeoutError +FAILED distributed/tests/test_client.py::test_run_rpc_error - TimeoutError +FAILED distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] - assert not True + + where True = all(. at 0x7f22015492f0>) += 6 failed, 2859 passed, 270 skipped, 221 deselected, 15 xfailed, 8 xpassed, 1 rerun in 4756.13s (1:19:16) = +*** END OF RUN 1: NOT ALL TESTS HAVE YET PASSED/XFAILED *** +*** STARTING RUN 2: python3.12 -m pytest --pyargs distributed --verbose --color=no --timeout-method=signal --timeout=300 -m not avoid_ci -rfE --last-failed --last-failed-no-failures none --ignore=distributed/comm/tests/test_comms.py --ignore=distributed/comm/tests/test_ws.py --ignore=distributed/deploy/tests/test_adaptive.py --ignore=distributed/deploy/tests/test_local.py --ignore=distributed/deploy/tests/test_slow_adaptive.py --ignore=distributed/deploy/tests/test_spec_cluster.py --deselect=distributed/cli/tests/test_dask_scheduler.py::test_no_dashboard --deselect=distributed/deploy/tests/test_local.py::test_localcluster_get_client --deselect=distributed/deploy/tests/test_old_ssh.py::test_cluster --deselect=distributed/deploy/tests/test_old_ssh.py::test_old_ssh_nprocs_renamed_to_n_workers --deselect=distributed/deploy/tests/test_old_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/deploy/tests/test_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/http/tests/test_core.py::test_prometheus_api_doc --deselect=distributed/tests/test_init.py::test_git_revision --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout_returned --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_jupyter.py::test_shutsdown_cleanly --deselect=distributed/tests/test_profile.py::test_stack_overflow --deselect=distributed/tests/test_pubsub.py::test_client_worker --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_spill.py::test_spillbuffer_oserror --deselect=distributed/tests/test_steal.py::test_steal_twice --deselect=distributed/tests/test_utils_test.py::test_cluster --deselect=distributed/tests/test_variable.py::test_variable_in_task --deselect=distributed/tests/test_worker.py::test_process_executor_kills_process --deselect=distributed/tests/test_worker_memory.py::test_fail_to_pickle_execute_1 --deselect=distributed/tests/test_worker_state_machine.py::test_task_state_instance_are_garbage_collected --deselect=distributed/protocol/tests/test_protocol.py::test_deeply_nested_structures --deselect=distributed/protocol/tests/test_serialize.py::test_deeply_nested_structures --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_spec.py::test_errors --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/dashboard/tests/test_scheduler_bokeh.py::test_counters --deselect=distributed/dashboard/tests/test_worker_bokeh.py::test_counters --deselect=distributed/deploy/tests/test_local.py::test_adapt_then_manual --deselect=distributed/deploy/tests/test_local.py::test_async_with --deselect=distributed/deploy/tests/test_local.py::test_close_twice --deselect=distributed/deploy/tests/test_local.py::test_cluster_info_sync --deselect=distributed/deploy/tests/test_local.py::test_local_tls --deselect=distributed/deploy/tests/test_local.py::test_no_dangling_asyncio_tasks --deselect=distributed/deploy/tests/test_local.py::test_only_local_access --deselect=distributed/deploy/tests/test_local.py::test_remote_access --deselect=distributed/diagnostics/tests/test_progress_widgets.py::test_serializers --deselect=distributed/diagnostics/tests/test_scheduler_plugin.py::test_lifecycle --deselect=distributed/http/scheduler/tests/test_missing_bokeh.py::test_missing_bokeh --deselect=distributed/http/scheduler/tests/test_scheduler_http.py::test_metrics_when_prometheus_client_not_installed --deselect=distributed/protocol/tests/test_serialize.py::test_errors --deselect=distributed/tests/test_batched.py::test_BatchedSend --deselect=distributed/tests/test_batched.py::test_close_closed --deselect=distributed/tests/test_batched.py::test_close_twice --deselect=distributed/tests/test_batched.py::test_send_after_stream_start --deselect=distributed/tests/test_batched.py::test_send_before_close --deselect=distributed/tests/test_batched.py::test_send_before_start --deselect=distributed/tests/test_batched.py::test_sending_traffic_jam --deselect=distributed/tests/test_batched.py::test_serializers --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_async_with --deselect=distributed/tests/test_client.py::test_client_is_quiet_cluster_close --deselect=distributed/tests/test_client.py::test_dashboard_link_cluster --deselect=distributed/tests/test_client.py::test_dashboard_link_inproc --deselect=distributed/tests/test_client.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_client.py::test_mixing_clients_different_scheduler --deselect=distributed/tests/test_client.py::test_quiet_client_close --deselect=distributed/tests/test_client.py::test_rebalance_sync --deselect=distributed/tests/test_client.py::test_repr_localcluster --deselect=distributed/tests/test_client.py::test_security_loader --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_shutdown --deselect=distributed/tests/test_client.py::test_shutdown_is_quiet_with_cluster --deselect=distributed/tests/test_client.py::test_shutdown_localcluster --deselect=distributed/tests/test_client.py::test_shutdown_stops_callbacks --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_start_new_loop --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_use_running_loop --deselect=distributed/tests/test_core.py::test_close_fast_without_active_handlers --deselect=distributed/tests/test_core.py::test_close_grace_period_for_handlers --deselect=distributed/tests/test_core.py::test_close_properly --deselect=distributed/tests/test_core.py::test_compression --deselect=distributed/tests/test_core.py::test_connection_pool --deselect=distributed/tests/test_core.py::test_connection_pool_close_while_connecting --deselect=distributed/tests/test_core.py::test_connection_pool_detects_remote_close --deselect=distributed/tests/test_core.py::test_connection_pool_outside_cancellation --deselect=distributed/tests/test_core.py::test_connection_pool_remove --deselect=distributed/tests/test_core.py::test_connection_pool_respects_limit --deselect=distributed/tests/test_core.py::test_connection_pool_tls --deselect=distributed/tests/test_core.py::test_counters --deselect=distributed/tests/test_core.py::test_deserialize_error --deselect=distributed/tests/test_core.py::test_errors --deselect=distributed/tests/test_core.py::test_identity_inproc --deselect=distributed/tests/test_core.py::test_identity_tcp --deselect=distributed/tests/test_core.py::test_large_packets_inproc --deselect=distributed/tests/test_core.py::test_messages_are_ordered_bsend --deselect=distributed/tests/test_core.py::test_messages_are_ordered_raw --deselect=distributed/tests/test_core.py::test_ports --deselect=distributed/tests/test_core.py::test_rpc_default --deselect=distributed/tests/test_core.py::test_rpc_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_default --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_tcp --deselect=distributed/tests/test_core.py::test_rpc_serialization --deselect=distributed/tests/test_core.py::test_rpc_tcp --deselect=distributed/tests/test_core.py::test_rpc_tls --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_inproc --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_tcp --deselect=distributed/tests/test_core.py::test_send_recv_args --deselect=distributed/tests/test_core.py::test_send_recv_cancelled --deselect=distributed/tests/test_core.py::test_server --deselect=distributed/tests/test_core.py::test_server_comms_mark_active_handlers --deselect=distributed/tests/test_core.py::test_server_raises_on_blocked_handlers --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_locks.py::test_errors --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_worker_uses_same_host_as_nanny --deselect=distributed/tests/test_preload.py::test_failure_doesnt_crash_scheduler --deselect=distributed/tests/test_preload.py::test_preload_import_time --deselect=distributed/tests/test_preload.py::test_preload_manager_sequence --deselect=distributed/tests/test_preload.py::test_worker_preload_text --deselect=distributed/tests/test_scheduler.py::test_allowed_failures_config --deselect=distributed/tests/test_scheduler.py::test_async_context_manager --deselect=distributed/tests/test_scheduler.py::test_dashboard_host --deselect=distributed/tests/test_scheduler.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_scheduler.py::test_finished --deselect=distributed/tests/test_scheduler.py::test_multiple_listeners --deselect=distributed/tests/test_scheduler.py::test_no_dangling_asyncio_tasks --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_security.py::test_require_encryption --deselect=distributed/tests/test_security.py::test_tls_listen_connect --deselect=distributed/tests/test_security.py::test_tls_temporary_credentials_functional --deselect=distributed/tests/test_semaphore.py::test_threadpoolworkers_pick_correct_ioloop --deselect=distributed/tests/test_tls_functional.py::test_security_dict_input_no_security --deselect=distributed/tests/test_utils_test.py::test_ensure_no_new_clients --deselect=distributed/tests/test_utils_test.py::test_freeze_batched_send --deselect=distributed/tests/test_utils_test.py::test_locked_comm_drop_in_replacement --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_read --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_write --deselect=distributed/tests/test_worker.py::test_host_uses_scheduler_protocol --deselect=distributed/tests/test_worker.py::test_plugin_exception --deselect=distributed/tests/test_worker.py::test_plugin_internal_exception --deselect=distributed/tests/test_worker.py::test_plugin_multiple_exceptions --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker_client.py::test_dont_override_default_get --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_allowlist --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_protocols --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers_2 --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command_default --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_config --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_file --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_remote_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_scheduler_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_contact_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_error_during_startup --deselect=distributed/cli/tests/test_dask_worker.py::test_integer_names --deselect=distributed/cli/tests/test_dask_worker.py::test_listen_address_ipv6 --deselect=distributed/cli/tests/test_dask_worker.py::test_local_directory --deselect=distributed/cli/tests/test_dask_worker.py::test_memory_limit --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range_too_many_workers_raises --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_no_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_auto --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_expands_name --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_negative --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_preload_config --deselect=distributed/cli/tests/test_dask_worker.py::test_resources --deselect=distributed/cli/tests/test_dask_worker.py::test_respect_host_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_address_env --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_restart_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_stagger_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_signal_handling --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_works --deselect=distributed/cli/tests/test_dask_worker.py::test_timeout --deselect=distributed/cli/tests/test_dask_worker.py::test_worker_class --deselect=distributed/tests/test_config.py::test_logging_extended --deselect=distributed/tests/test_config.py::test_logging_file_config --deselect=distributed/tests/test_config.py::test_logging_mutual_exclusive --deselect=distributed/tests/test_config.py::test_logging_simple --deselect=distributed/tests/test_config.py::test_logging_simple_under_distributed --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_variable.py::test_variable_in_task +============================= test session starts ============================== +platform linux -- Python 3.12.6, pytest-8.3.3, pluggy-1.5.0 -- /usr/bin/python3.12 +cachedir: .pytest_cache +rootdir: /build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build +configfile: pyproject.toml +plugins: rerunfailures-12.0, typeguard-4.3.0, timeout-2.3.1, anyio-4.4.0 +timeout: 300.0s +timeout method: signal +timeout func_only: False +collecting ... collected 10 items / 4 deselected / 6 selected +run-last-failure: rerun previous 6 failures (skipped 143 files) + +distributed/shuffle/tests/test_rechunk.py::test_rechunk_method PASSED [ 16%] +distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] PASSED [ 33%] +distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] PASSED [ 50%] +distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] PASSED [ 66%] +distributed/tests/test_client.py::test_run_rpc_error PASSED [ 83%] +distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] FAILED [100%] + +=================================== FAILURES =================================== +___________________ test_last_in_first_out[queue on worker] ____________________ + +c = +s = +a = +pause = False + + @gen_blockable_cluster + async def test_last_in_first_out(c, s, a, pause): + async with block_worker(c, s, a, pause, 15, 5): + xs = [c.submit(slowinc, i, delay=0.05, key=f"x{i}") for i in range(5)] + ys = [c.submit(slowinc, xs[i], delay=0.05, key=f"y{i}") for i in range(5)] + zs = [c.submit(slowinc, ys[i], delay=0.05, key=f"z{i}") for i in range(5)] + + while not any(s.tasks[z.key].state == "memory" for z in zs): + await asyncio.sleep(0.01) +> assert not all(s.tasks[x.key].state == "memory" for x in xs) +E assert not True +E + where True = all(. at 0x7f74942dfb90>) + +distributed/tests/test_priorities.py:234: AssertionError +----------------------------- Captured stdout call ----------------------------- +Dumped cluster state to test_cluster_dump/test_last_in_first_out.yaml +----------------------------- Captured stderr call ----------------------------- +2024-10-22 07:48:23,715 - distributed.scheduler - INFO - State start +2024-10-22 07:48:23,727 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:39429 +2024-10-22 07:48:23,728 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34091/status +2024-10-22 07:48:23,728 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 07:48:23,746 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:33631 +2024-10-22 07:48:23,746 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:33631 +2024-10-22 07:48:23,746 - distributed.worker - INFO - Worker name: 0 +2024-10-22 07:48:23,746 - distributed.worker - INFO - dashboard at: 127.0.0.1:33037 +2024-10-22 07:48:23,746 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:39429 +2024-10-22 07:48:23,747 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:48:23,747 - distributed.worker - INFO - Threads: 1 +2024-10-22 07:48:23,747 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 07:48:23,747 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-lkucf07d +2024-10-22 07:48:23,747 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:48:23,750 - distributed.scheduler - INFO - Register worker +2024-10-22 07:48:23,751 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:33631 +2024-10-22 07:48:23,751 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54120 +2024-10-22 07:48:23,752 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 07:48:23,752 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:39429 +2024-10-22 07:48:23,761 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:48:23,762 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:39429 +2024-10-22 07:48:23,772 - distributed.scheduler - INFO - Receive client connection: Client-ab10e344-8fd4-11ef-8fe1-631a890fd210 +2024-10-22 07:48:23,781 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54136 +2024-10-22 07:48:25,373 - distributed.scheduler - INFO - Remove client Client-ab10e344-8fd4-11ef-8fe1-631a890fd210 +2024-10-22 07:48:25,374 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:54136; closing. +2024-10-22 07:48:25,374 - distributed.scheduler - INFO - Remove client Client-ab10e344-8fd4-11ef-8fe1-631a890fd210 +2024-10-22 07:48:25,377 - distributed.scheduler - INFO - Close client connection: Client-ab10e344-8fd4-11ef-8fe1-631a890fd210 +2024-10-22 07:48:25,379 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:33631. Reason: worker-close +2024-10-22 07:48:25,381 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 07:48:25,383 - distributed.core - INFO - Connection to tcp://127.0.0.1:39429 has been closed. +2024-10-22 07:48:25,391 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:54120; closing. +2024-10-22 07:48:25,392 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729532905.3921595') +2024-10-22 07:48:25,392 - distributed.scheduler - INFO - Lost all workers +2024-10-22 07:48:25,394 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 07:48:25,394 - distributed.scheduler - INFO - Scheduler closing all comms +============================= slowest 20 durations ============================= +18.24s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_method +15.90s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] +7.32s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] +1.70s call distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.99s call distributed/tests/test_client.py::test_run_rpc_error +0.90s call distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] +0.00s setup distributed/shuffle/tests/test_rechunk.py::test_rechunk_method +0.00s setup distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] +0.00s setup distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] +0.00s teardown distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.00s setup distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] +0.00s teardown distributed/tests/test_cancelled_state.py::test_deadlock_cancelled_after_inflight_before_gather_from_worker[True-cancelled] +0.00s teardown distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] +0.00s teardown distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x3-chunks3] +0.00s teardown distributed/shuffle/tests/test_rechunk.py::test_rechunk_method +0.00s setup distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.00s teardown distributed/tests/test_client.py::test_run_rpc_error +0.00s setup distributed/tests/test_client.py::test_run_rpc_error +=========================== short test summary info ============================ +FAILED distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] - assert not True + + where True = all(. at 0x7f74942dfb90>) +================== 1 failed, 5 passed, 4 deselected in 46.76s ================== +*** END OF RUN 2: NOT ALL TESTS HAVE YET PASSED/XFAILED *** +*** STARTING RUN 3: python3.12 -m pytest --pyargs distributed --verbose --color=no --timeout-method=signal --timeout=300 -m not avoid_ci -rfE --last-failed --last-failed-no-failures none --ignore=distributed/comm/tests/test_comms.py --ignore=distributed/comm/tests/test_ws.py --ignore=distributed/deploy/tests/test_adaptive.py --ignore=distributed/deploy/tests/test_local.py --ignore=distributed/deploy/tests/test_slow_adaptive.py --ignore=distributed/deploy/tests/test_spec_cluster.py --deselect=distributed/cli/tests/test_dask_scheduler.py::test_no_dashboard --deselect=distributed/deploy/tests/test_local.py::test_localcluster_get_client --deselect=distributed/deploy/tests/test_old_ssh.py::test_cluster --deselect=distributed/deploy/tests/test_old_ssh.py::test_old_ssh_nprocs_renamed_to_n_workers --deselect=distributed/deploy/tests/test_old_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/deploy/tests/test_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/http/tests/test_core.py::test_prometheus_api_doc --deselect=distributed/tests/test_init.py::test_git_revision --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout_returned --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_jupyter.py::test_shutsdown_cleanly --deselect=distributed/tests/test_profile.py::test_stack_overflow --deselect=distributed/tests/test_pubsub.py::test_client_worker --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_spill.py::test_spillbuffer_oserror --deselect=distributed/tests/test_steal.py::test_steal_twice --deselect=distributed/tests/test_utils_test.py::test_cluster --deselect=distributed/tests/test_variable.py::test_variable_in_task --deselect=distributed/tests/test_worker.py::test_process_executor_kills_process --deselect=distributed/tests/test_worker_memory.py::test_fail_to_pickle_execute_1 --deselect=distributed/tests/test_worker_state_machine.py::test_task_state_instance_are_garbage_collected --deselect=distributed/protocol/tests/test_protocol.py::test_deeply_nested_structures --deselect=distributed/protocol/tests/test_serialize.py::test_deeply_nested_structures --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_spec.py::test_errors --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/dashboard/tests/test_scheduler_bokeh.py::test_counters --deselect=distributed/dashboard/tests/test_worker_bokeh.py::test_counters --deselect=distributed/deploy/tests/test_local.py::test_adapt_then_manual --deselect=distributed/deploy/tests/test_local.py::test_async_with --deselect=distributed/deploy/tests/test_local.py::test_close_twice --deselect=distributed/deploy/tests/test_local.py::test_cluster_info_sync --deselect=distributed/deploy/tests/test_local.py::test_local_tls --deselect=distributed/deploy/tests/test_local.py::test_no_dangling_asyncio_tasks --deselect=distributed/deploy/tests/test_local.py::test_only_local_access --deselect=distributed/deploy/tests/test_local.py::test_remote_access --deselect=distributed/diagnostics/tests/test_progress_widgets.py::test_serializers --deselect=distributed/diagnostics/tests/test_scheduler_plugin.py::test_lifecycle --deselect=distributed/http/scheduler/tests/test_missing_bokeh.py::test_missing_bokeh --deselect=distributed/http/scheduler/tests/test_scheduler_http.py::test_metrics_when_prometheus_client_not_installed --deselect=distributed/protocol/tests/test_serialize.py::test_errors --deselect=distributed/tests/test_batched.py::test_BatchedSend --deselect=distributed/tests/test_batched.py::test_close_closed --deselect=distributed/tests/test_batched.py::test_close_twice --deselect=distributed/tests/test_batched.py::test_send_after_stream_start --deselect=distributed/tests/test_batched.py::test_send_before_close --deselect=distributed/tests/test_batched.py::test_send_before_start --deselect=distributed/tests/test_batched.py::test_sending_traffic_jam --deselect=distributed/tests/test_batched.py::test_serializers --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_async_with --deselect=distributed/tests/test_client.py::test_client_is_quiet_cluster_close --deselect=distributed/tests/test_client.py::test_dashboard_link_cluster --deselect=distributed/tests/test_client.py::test_dashboard_link_inproc --deselect=distributed/tests/test_client.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_client.py::test_mixing_clients_different_scheduler --deselect=distributed/tests/test_client.py::test_quiet_client_close --deselect=distributed/tests/test_client.py::test_rebalance_sync --deselect=distributed/tests/test_client.py::test_repr_localcluster --deselect=distributed/tests/test_client.py::test_security_loader --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_shutdown --deselect=distributed/tests/test_client.py::test_shutdown_is_quiet_with_cluster --deselect=distributed/tests/test_client.py::test_shutdown_localcluster --deselect=distributed/tests/test_client.py::test_shutdown_stops_callbacks --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_start_new_loop --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_use_running_loop --deselect=distributed/tests/test_core.py::test_close_fast_without_active_handlers --deselect=distributed/tests/test_core.py::test_close_grace_period_for_handlers --deselect=distributed/tests/test_core.py::test_close_properly --deselect=distributed/tests/test_core.py::test_compression --deselect=distributed/tests/test_core.py::test_connection_pool --deselect=distributed/tests/test_core.py::test_connection_pool_close_while_connecting --deselect=distributed/tests/test_core.py::test_connection_pool_detects_remote_close --deselect=distributed/tests/test_core.py::test_connection_pool_outside_cancellation --deselect=distributed/tests/test_core.py::test_connection_pool_remove --deselect=distributed/tests/test_core.py::test_connection_pool_respects_limit --deselect=distributed/tests/test_core.py::test_connection_pool_tls --deselect=distributed/tests/test_core.py::test_counters --deselect=distributed/tests/test_core.py::test_deserialize_error --deselect=distributed/tests/test_core.py::test_errors --deselect=distributed/tests/test_core.py::test_identity_inproc --deselect=distributed/tests/test_core.py::test_identity_tcp --deselect=distributed/tests/test_core.py::test_large_packets_inproc --deselect=distributed/tests/test_core.py::test_messages_are_ordered_bsend --deselect=distributed/tests/test_core.py::test_messages_are_ordered_raw --deselect=distributed/tests/test_core.py::test_ports --deselect=distributed/tests/test_core.py::test_rpc_default --deselect=distributed/tests/test_core.py::test_rpc_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_default --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_tcp --deselect=distributed/tests/test_core.py::test_rpc_serialization --deselect=distributed/tests/test_core.py::test_rpc_tcp --deselect=distributed/tests/test_core.py::test_rpc_tls --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_inproc --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_tcp --deselect=distributed/tests/test_core.py::test_send_recv_args --deselect=distributed/tests/test_core.py::test_send_recv_cancelled --deselect=distributed/tests/test_core.py::test_server --deselect=distributed/tests/test_core.py::test_server_comms_mark_active_handlers --deselect=distributed/tests/test_core.py::test_server_raises_on_blocked_handlers --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_locks.py::test_errors --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_worker_uses_same_host_as_nanny --deselect=distributed/tests/test_preload.py::test_failure_doesnt_crash_scheduler --deselect=distributed/tests/test_preload.py::test_preload_import_time --deselect=distributed/tests/test_preload.py::test_preload_manager_sequence --deselect=distributed/tests/test_preload.py::test_worker_preload_text --deselect=distributed/tests/test_scheduler.py::test_allowed_failures_config --deselect=distributed/tests/test_scheduler.py::test_async_context_manager --deselect=distributed/tests/test_scheduler.py::test_dashboard_host --deselect=distributed/tests/test_scheduler.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_scheduler.py::test_finished --deselect=distributed/tests/test_scheduler.py::test_multiple_listeners --deselect=distributed/tests/test_scheduler.py::test_no_dangling_asyncio_tasks --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_security.py::test_require_encryption --deselect=distributed/tests/test_security.py::test_tls_listen_connect --deselect=distributed/tests/test_security.py::test_tls_temporary_credentials_functional --deselect=distributed/tests/test_semaphore.py::test_threadpoolworkers_pick_correct_ioloop --deselect=distributed/tests/test_tls_functional.py::test_security_dict_input_no_security --deselect=distributed/tests/test_utils_test.py::test_ensure_no_new_clients --deselect=distributed/tests/test_utils_test.py::test_freeze_batched_send --deselect=distributed/tests/test_utils_test.py::test_locked_comm_drop_in_replacement --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_read --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_write --deselect=distributed/tests/test_worker.py::test_host_uses_scheduler_protocol --deselect=distributed/tests/test_worker.py::test_plugin_exception --deselect=distributed/tests/test_worker.py::test_plugin_internal_exception --deselect=distributed/tests/test_worker.py::test_plugin_multiple_exceptions --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker_client.py::test_dont_override_default_get --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_allowlist --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_protocols --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers_2 --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command_default --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_config --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_file --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_remote_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_scheduler_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_contact_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_error_during_startup --deselect=distributed/cli/tests/test_dask_worker.py::test_integer_names --deselect=distributed/cli/tests/test_dask_worker.py::test_listen_address_ipv6 --deselect=distributed/cli/tests/test_dask_worker.py::test_local_directory --deselect=distributed/cli/tests/test_dask_worker.py::test_memory_limit --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range_too_many_workers_raises --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_no_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_auto --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_expands_name --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_negative --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_preload_config --deselect=distributed/cli/tests/test_dask_worker.py::test_resources --deselect=distributed/cli/tests/test_dask_worker.py::test_respect_host_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_address_env --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_restart_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_stagger_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_signal_handling --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_works --deselect=distributed/cli/tests/test_dask_worker.py::test_timeout --deselect=distributed/cli/tests/test_dask_worker.py::test_worker_class --deselect=distributed/tests/test_config.py::test_logging_extended --deselect=distributed/tests/test_config.py::test_logging_file_config --deselect=distributed/tests/test_config.py::test_logging_mutual_exclusive --deselect=distributed/tests/test_config.py::test_logging_simple --deselect=distributed/tests/test_config.py::test_logging_simple_under_distributed --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_variable.py::test_variable_in_task +============================= test session starts ============================== +platform linux -- Python 3.12.6, pytest-8.3.3, pluggy-1.5.0 -- /usr/bin/python3.12 +cachedir: .pytest_cache +rootdir: /build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build +configfile: pyproject.toml +plugins: rerunfailures-12.0, typeguard-4.3.0, timeout-2.3.1, anyio-4.4.0 +timeout: 300.0s +timeout method: signal +timeout func_only: False +collecting ... collected 1 item +run-last-failure: rerun previous 1 failure (skipped 146 files) + +distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] FAILED [100%] + +=================================== FAILURES =================================== +___________________ test_last_in_first_out[queue on worker] ____________________ + +c = +s = +a = +pause = False + + @gen_blockable_cluster + async def test_last_in_first_out(c, s, a, pause): + async with block_worker(c, s, a, pause, 15, 5): + xs = [c.submit(slowinc, i, delay=0.05, key=f"x{i}") for i in range(5)] + ys = [c.submit(slowinc, xs[i], delay=0.05, key=f"y{i}") for i in range(5)] + zs = [c.submit(slowinc, ys[i], delay=0.05, key=f"z{i}") for i in range(5)] + + while not any(s.tasks[z.key].state == "memory" for z in zs): + await asyncio.sleep(0.01) +> assert not all(s.tasks[x.key].state == "memory" for x in xs) +E assert not True +E + where True = all(. at 0x7f01f9cfcad0>) + +distributed/tests/test_priorities.py:234: AssertionError +----------------------------- Captured stdout call ----------------------------- +Dumped cluster state to test_cluster_dump/test_last_in_first_out.yaml +----------------------------- Captured stderr call ----------------------------- +2024-10-22 07:48:28,657 - distributed.http.proxy - INFO - To route to workers diagnostics web server please install jupyter-server-proxy: python -m pip install jupyter-server-proxy +2024-10-22 07:48:28,663 - distributed.scheduler - INFO - State start +2024-10-22 07:48:28,670 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:38255 +2024-10-22 07:48:28,671 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34479/status +2024-10-22 07:48:28,671 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2024-10-22 07:48:28,679 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:46809 +2024-10-22 07:48:28,680 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:46809 +2024-10-22 07:48:28,680 - distributed.worker - INFO - Worker name: 0 +2024-10-22 07:48:28,680 - distributed.worker - INFO - dashboard at: 127.0.0.1:34407 +2024-10-22 07:48:28,680 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:38255 +2024-10-22 07:48:28,680 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:48:28,680 - distributed.worker - INFO - Threads: 1 +2024-10-22 07:48:28,680 - distributed.worker - INFO - Memory: 78.53 GiB +2024-10-22 07:48:28,680 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-xw7lruxg +2024-10-22 07:48:28,680 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:48:29,304 - distributed.scheduler - INFO - Register worker +2024-10-22 07:48:29,305 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:46809 +2024-10-22 07:48:29,305 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:39666 +2024-10-22 07:48:29,306 - distributed.worker - INFO - Starting Worker plugin shuffle +2024-10-22 07:48:29,306 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:38255 +2024-10-22 07:48:29,306 - distributed.worker - INFO - ------------------------------------------------- +2024-10-22 07:48:29,307 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:38255 +2024-10-22 07:48:29,313 - distributed.scheduler - INFO - Receive client connection: Client-ae5f0c43-8fd4-11ef-8449-51ebeaa3d26a +2024-10-22 07:48:29,314 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:39670 +2024-10-22 07:48:30,489 - distributed.scheduler - INFO - Remove client Client-ae5f0c43-8fd4-11ef-8449-51ebeaa3d26a +2024-10-22 07:48:30,490 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:39670; closing. +2024-10-22 07:48:30,491 - distributed.scheduler - INFO - Remove client Client-ae5f0c43-8fd4-11ef-8449-51ebeaa3d26a +2024-10-22 07:48:30,493 - distributed.scheduler - INFO - Close client connection: Client-ae5f0c43-8fd4-11ef-8449-51ebeaa3d26a +2024-10-22 07:48:30,496 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:46809. Reason: worker-close +2024-10-22 07:48:30,498 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2024-10-22 07:48:30,500 - distributed.core - INFO - Connection to tcp://127.0.0.1:38255 has been closed. +2024-10-22 07:48:30,523 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:39666; closing. +2024-10-22 07:48:30,523 - distributed.scheduler - INFO - Remove worker (stimulus_id='handle-worker-cleanup-1729532910.5238948') +2024-10-22 07:48:30,524 - distributed.scheduler - INFO - Lost all workers +2024-10-22 07:48:30,538 - distributed.scheduler - INFO - Scheduler closing due to unknown reason... +2024-10-22 07:48:30,538 - distributed.scheduler - INFO - Scheduler closing all comms +============================= slowest 20 durations ============================= +1.95s call distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.00s setup distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.00s teardown distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +=========================== short test summary info ============================ +FAILED distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] - assert not True + + where True = all(. at 0x7f01f9cfcad0>) +============================== 1 failed in 2.54s =============================== +*** END OF RUN 3: NOT ALL TESTS HAVE YET PASSED/XFAILED *** +*** STARTING RUN 4: python3.12 -m pytest --pyargs distributed --verbose --color=no --timeout-method=signal --timeout=300 -m not avoid_ci -rfE --last-failed --last-failed-no-failures none --ignore=distributed/comm/tests/test_comms.py --ignore=distributed/comm/tests/test_ws.py --ignore=distributed/deploy/tests/test_adaptive.py --ignore=distributed/deploy/tests/test_local.py --ignore=distributed/deploy/tests/test_slow_adaptive.py --ignore=distributed/deploy/tests/test_spec_cluster.py --deselect=distributed/cli/tests/test_dask_scheduler.py::test_no_dashboard --deselect=distributed/deploy/tests/test_local.py::test_localcluster_get_client --deselect=distributed/deploy/tests/test_old_ssh.py::test_cluster --deselect=distributed/deploy/tests/test_old_ssh.py::test_old_ssh_nprocs_renamed_to_n_workers --deselect=distributed/deploy/tests/test_old_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/deploy/tests/test_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/http/tests/test_core.py::test_prometheus_api_doc --deselect=distributed/tests/test_init.py::test_git_revision --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout_returned --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_jupyter.py::test_shutsdown_cleanly --deselect=distributed/tests/test_profile.py::test_stack_overflow --deselect=distributed/tests/test_pubsub.py::test_client_worker --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_spill.py::test_spillbuffer_oserror --deselect=distributed/tests/test_steal.py::test_steal_twice --deselect=distributed/tests/test_utils_test.py::test_cluster --deselect=distributed/tests/test_variable.py::test_variable_in_task --deselect=distributed/tests/test_worker.py::test_process_executor_kills_process --deselect=distributed/tests/test_worker_memory.py::test_fail_to_pickle_execute_1 --deselect=distributed/tests/test_worker_state_machine.py::test_task_state_instance_are_garbage_collected --deselect=distributed/protocol/tests/test_protocol.py::test_deeply_nested_structures --deselect=distributed/protocol/tests/test_serialize.py::test_deeply_nested_structures --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_spec.py::test_errors --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/dashboard/tests/test_scheduler_bokeh.py::test_counters --deselect=distributed/dashboard/tests/test_worker_bokeh.py::test_counters --deselect=distributed/deploy/tests/test_local.py::test_adapt_then_manual --deselect=distributed/deploy/tests/test_local.py::test_async_with --deselect=distributed/deploy/tests/test_local.py::test_close_twice --deselect=distributed/deploy/tests/test_local.py::test_cluster_info_sync --deselect=distributed/deploy/tests/test_local.py::test_local_tls --deselect=distributed/deploy/tests/test_local.py::test_no_dangling_asyncio_tasks --deselect=distributed/deploy/tests/test_local.py::test_only_local_access --deselect=distributed/deploy/tests/test_local.py::test_remote_access --deselect=distributed/diagnostics/tests/test_progress_widgets.py::test_serializers --deselect=distributed/diagnostics/tests/test_scheduler_plugin.py::test_lifecycle --deselect=distributed/http/scheduler/tests/test_missing_bokeh.py::test_missing_bokeh --deselect=distributed/http/scheduler/tests/test_scheduler_http.py::test_metrics_when_prometheus_client_not_installed --deselect=distributed/protocol/tests/test_serialize.py::test_errors --deselect=distributed/tests/test_batched.py::test_BatchedSend --deselect=distributed/tests/test_batched.py::test_close_closed --deselect=distributed/tests/test_batched.py::test_close_twice --deselect=distributed/tests/test_batched.py::test_send_after_stream_start --deselect=distributed/tests/test_batched.py::test_send_before_close --deselect=distributed/tests/test_batched.py::test_send_before_start --deselect=distributed/tests/test_batched.py::test_sending_traffic_jam --deselect=distributed/tests/test_batched.py::test_serializers --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_async_with --deselect=distributed/tests/test_client.py::test_client_is_quiet_cluster_close --deselect=distributed/tests/test_client.py::test_dashboard_link_cluster --deselect=distributed/tests/test_client.py::test_dashboard_link_inproc --deselect=distributed/tests/test_client.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_client.py::test_mixing_clients_different_scheduler --deselect=distributed/tests/test_client.py::test_quiet_client_close --deselect=distributed/tests/test_client.py::test_rebalance_sync --deselect=distributed/tests/test_client.py::test_repr_localcluster --deselect=distributed/tests/test_client.py::test_security_loader --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_shutdown --deselect=distributed/tests/test_client.py::test_shutdown_is_quiet_with_cluster --deselect=distributed/tests/test_client.py::test_shutdown_localcluster --deselect=distributed/tests/test_client.py::test_shutdown_stops_callbacks --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_start_new_loop --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_use_running_loop --deselect=distributed/tests/test_core.py::test_close_fast_without_active_handlers --deselect=distributed/tests/test_core.py::test_close_grace_period_for_handlers --deselect=distributed/tests/test_core.py::test_close_properly --deselect=distributed/tests/test_core.py::test_compression --deselect=distributed/tests/test_core.py::test_connection_pool --deselect=distributed/tests/test_core.py::test_connection_pool_close_while_connecting --deselect=distributed/tests/test_core.py::test_connection_pool_detects_remote_close --deselect=distributed/tests/test_core.py::test_connection_pool_outside_cancellation --deselect=distributed/tests/test_core.py::test_connection_pool_remove --deselect=distributed/tests/test_core.py::test_connection_pool_respects_limit --deselect=distributed/tests/test_core.py::test_connection_pool_tls --deselect=distributed/tests/test_core.py::test_counters --deselect=distributed/tests/test_core.py::test_deserialize_error --deselect=distributed/tests/test_core.py::test_errors --deselect=distributed/tests/test_core.py::test_identity_inproc --deselect=distributed/tests/test_core.py::test_identity_tcp --deselect=distributed/tests/test_core.py::test_large_packets_inproc --deselect=distributed/tests/test_core.py::test_messages_are_ordered_bsend --deselect=distributed/tests/test_core.py::test_messages_are_ordered_raw --deselect=distributed/tests/test_core.py::test_ports --deselect=distributed/tests/test_core.py::test_rpc_default --deselect=distributed/tests/test_core.py::test_rpc_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_default --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_tcp --deselect=distributed/tests/test_core.py::test_rpc_serialization --deselect=distributed/tests/test_core.py::test_rpc_tcp --deselect=distributed/tests/test_core.py::test_rpc_tls --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_inproc --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_tcp --deselect=distributed/tests/test_core.py::test_send_recv_args --deselect=distributed/tests/test_core.py::test_send_recv_cancelled --deselect=distributed/tests/test_core.py::test_server --deselect=distributed/tests/test_core.py::test_server_comms_mark_active_handlers --deselect=distributed/tests/test_core.py::test_server_raises_on_blocked_handlers --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_locks.py::test_errors --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_worker_uses_same_host_as_nanny --deselect=distributed/tests/test_preload.py::test_failure_doesnt_crash_scheduler --deselect=distributed/tests/test_preload.py::test_preload_import_time --deselect=distributed/tests/test_preload.py::test_preload_manager_sequence --deselect=distributed/tests/test_preload.py::test_worker_preload_text --deselect=distributed/tests/test_scheduler.py::test_allowed_failures_config --deselect=distributed/tests/test_scheduler.py::test_async_context_manager --deselect=distributed/tests/test_scheduler.py::test_dashboard_host --deselect=distributed/tests/test_scheduler.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_scheduler.py::test_finished --deselect=distributed/tests/test_scheduler.py::test_multiple_listeners --deselect=distributed/tests/test_scheduler.py::test_no_dangling_asyncio_tasks --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_security.py::test_require_encryption --deselect=distributed/tests/test_security.py::test_tls_listen_connect --deselect=distributed/tests/test_security.py::test_tls_temporary_credentials_functional --deselect=distributed/tests/test_semaphore.py::test_threadpoolworkers_pick_correct_ioloop --deselect=distributed/tests/test_tls_functional.py::test_security_dict_input_no_security --deselect=distributed/tests/test_utils_test.py::test_ensure_no_new_clients --deselect=distributed/tests/test_utils_test.py::test_freeze_batched_send --deselect=distributed/tests/test_utils_test.py::test_locked_comm_drop_in_replacement --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_read --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_write --deselect=distributed/tests/test_worker.py::test_host_uses_scheduler_protocol --deselect=distributed/tests/test_worker.py::test_plugin_exception --deselect=distributed/tests/test_worker.py::test_plugin_internal_exception --deselect=distributed/tests/test_worker.py::test_plugin_multiple_exceptions --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker_client.py::test_dont_override_default_get --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_allowlist --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_protocols --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers_2 --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command_default --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_config --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_file --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_remote_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_scheduler_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_contact_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_error_during_startup --deselect=distributed/cli/tests/test_dask_worker.py::test_integer_names --deselect=distributed/cli/tests/test_dask_worker.py::test_listen_address_ipv6 --deselect=distributed/cli/tests/test_dask_worker.py::test_local_directory --deselect=distributed/cli/tests/test_dask_worker.py::test_memory_limit --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range_too_many_workers_raises --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_no_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_auto --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_expands_name --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_negative --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_preload_config --deselect=distributed/cli/tests/test_dask_worker.py::test_resources --deselect=distributed/cli/tests/test_dask_worker.py::test_respect_host_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_address_env --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_restart_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_stagger_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_signal_handling --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_works --deselect=distributed/cli/tests/test_dask_worker.py::test_timeout --deselect=distributed/cli/tests/test_dask_worker.py::test_worker_class --deselect=distributed/tests/test_config.py::test_logging_extended --deselect=distributed/tests/test_config.py::test_logging_file_config --deselect=distributed/tests/test_config.py::test_logging_mutual_exclusive --deselect=distributed/tests/test_config.py::test_logging_simple --deselect=distributed/tests/test_config.py::test_logging_simple_under_distributed --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_variable.py::test_variable_in_task +============================= test session starts ============================== +platform linux -- Python 3.12.6, pytest-8.3.3, pluggy-1.5.0 -- /usr/bin/python3.12 +cachedir: .pytest_cache +rootdir: /build/reproducible-path/dask.distributed-2024.5.2+ds.1/.pybuild/cpython3_3.12_distributed/build +configfile: pyproject.toml +plugins: rerunfailures-12.0, typeguard-4.3.0, timeout-2.3.1, anyio-4.4.0 +timeout: 300.0s +timeout method: signal +timeout func_only: False +collecting ... collected 1 item +run-last-failure: rerun previous 1 failure (skipped 146 files) + +distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] PASSED [100%] + +============================= slowest 20 durations ============================= +1.90s call distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.00s setup distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +0.00s teardown distributed/tests/test_priorities.py::test_last_in_first_out[queue on worker] +============================== 1 passed in 2.72s =============================== +*** END OF RUN 4: ALL TESTS RUN HAVE NOW PASSED/XFAILED *** create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=pybuild dh_prep -O--buildsystem=pybuild @@ -6843,9 +9138,9 @@ removed 'debian/python-distributed-doc/usr/share/doc/python3-distributed/html/_static/vendor/fontawesome/5.13.0/LICENSE.txt' make[1]: Leaving directory '/build/reproducible-path/dask.distributed-2024.5.2+ds.1' dh_sphinxdoc -O--buildsystem=pybuild -dh_sphinxdoc: warning: ignoring unknown JavaScript code: debian/python-distributed-doc/usr/share/doc/python3-distributed/html/_static/design-tabs.js -dh_sphinxdoc: warning: ignoring unknown JavaScript code: debian/python-distributed-doc/usr/share/doc/python3-distributed/html/_static/sphinx-book-theme.12a9622fbb08dcb3a2a40b2c02b83a57.js dh_sphinxdoc: warning: ignoring unknown JavaScript code: debian/python-distributed-doc/usr/share/doc/python3-distributed/html/_static/js/index.be7d3bbb2ef33a8344ce.js +dh_sphinxdoc: warning: ignoring unknown JavaScript code: debian/python-distributed-doc/usr/share/doc/python3-distributed/html/_static/sphinx-book-theme.12a9622fbb08dcb3a2a40b2c02b83a57.js +dh_sphinxdoc: warning: ignoring unknown JavaScript code: debian/python-distributed-doc/usr/share/doc/python3-distributed/html/_static/design-tabs.js dh_installchangelogs -O--buildsystem=pybuild dh_python3 -O--buildsystem=pybuild dh_installsystemduser -O--buildsystem=pybuild @@ -6872,12 +9167,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: not including original source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/1538210/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/2912599 and its subdirectories -I: Current time: Sun Nov 23 10:38:23 -12 2025 -I: pbuilder-time-stamp: 1763937503 +I: removing directory /srv/workspace/pbuilder/1538210 and its subdirectories +I: Current time: Tue Oct 22 07:49:32 +14 2024 +I: pbuilder-time-stamp: 1729532972