{"msg":"Running step start_crdb","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:06.122941975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Starting CRDB","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:06.123672156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"cockroach temporary directory: /var/tmp/omicron_tmp/.tmpPFWBYc","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:06.183788502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"cockroach: copying from seed tarball (/var/tmp/omicron_tmp/crdb-base-build/fd6cb9b3b8228bca1b9a6434e5bd596c143610f1584e14fb09b8029343bfa9a2.tar) to storage directory (/var/tmp/omicron_tmp/.tmpPFWBYc/data)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:06.183870675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"cockroach command line: cockroach start-single-node --insecure --http-addr=:0 --store=path=/var/tmp/omicron_tmp/.tmpPFWBYc/data,ballast-size=0 --listen-addr [::1]:0 --listening-url-file /var/tmp/omicron_tmp/.tmpPFWBYc/listen-url --max-sql-memory 256MiB","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:06.22743663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"cockroach environment: BUILDOMAT_JOB_ID=01KJVJ8F1T1C8BG548GQB09ZVN BUILDOMAT_TASK_ID=4 CARGO=/home/build/.rustup/toolchains/1.91.1-x86_64-unknown-illumos/bin/cargo CARGO_HOME=/home/build/.cargo CARGO_INCREMENTAL=0 CARGO_MANIFEST_DIR=/work/oxidecomputer/omicron/nexus CARGO_PKG_AUTHORS= CARGO_PKG_DESCRIPTION= CARGO_PKG_HOMEPAGE= CARGO_PKG_LICENSE=MPL-2.0 CARGO_PKG_LICENSE_FILE= CARGO_PKG_NAME=omicron-nexus CARGO_PKG_REPOSITORY= CARGO_PKG_RUST_VERSION= CARGO_PKG_VERSION=0.1.0 CARGO_PKG_VERSION_MAJOR=0 CARGO_PKG_VERSION_MINOR=1 CARGO_PKG_VERSION_PATCH=0 CARGO_PKG_VERSION_PRE= CARGO_TERM_COLOR=always CI=true CRDB_SEED_TAR=/var/tmp/omicron_tmp/crdb-base-build/fd6cb9b3b8228bca1b9a6434e5bd596c143610f1584e14fb09b8029343bfa9a2.tar GITHUB_REPOSITORY=oxidecomputer/omicron GITHUB_SHA=1c76132ffe5de6593276543ceb5c830b9f21e4e1 GOTRACEBACK=crash HOME=/home/build LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 LD_LIBRARY_PATH=/work/oxidecomputer/omicron/target/debug/build/aws-lc-sys-b2991dcfe0032b19/out:/work/oxidecomputer/omicron/target/debug/build/blake3-df20c6b18d3956e5/out:/work/oxidecomputer/omicron/target/debug/build/bzip2-sys-8b0f050bf31de6f5/out/lib:/work/oxidecomputer/omicron/target/debug/build/libgit2-sys-fc1bb2730a9a80d7/out/build:/work/oxidecomputer/omicron/target/debug/build/psm-e3b3fd8e545b20fd/out:/work/oxidecomputer/omicron/target/debug/build/ring-882060e0ad2a422c/out:/work/oxidecomputer/omicron/target/debug/build/tofino-d064271f908edbf6/out:/work/oxidecomputer/omicron/target/debug/build/zstd-sys-54b41a4eb78e9d96/out:/work/oxidecomputer/omicron/target/debug/deps:/work/oxidecomputer/omicron/target/debug:/home/build/.rustup/toolchains/1.91.1-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib:/home/build/.rustup/toolchains/1.91.1-x86_64-unknown-illumos/lib LOGNAME=build NEXTEST=1 NEXTEST_ATTEMPT=1 NEXTEST_ATTEMPT_ID=ec1b80b1-e223-4a9a-80e8-f587c0ea1d10:omicron-nexus::test_all$integration_tests::volume_management::test_upstairs_notify_downstairs_client_stops NEXTEST_BINARY_ID=omicron-nexus::test_all NEXTEST_BIN_EXE_nexus=/work/oxidecomputer/omicron/target/debug/nexus NEXTEST_BIN_EXE_schema-updater=/work/oxidecomputer/omicron/target/debug/schema-updater NEXTEST_BIN_EXE_schema_updater=/work/oxidecomputer/omicron/target/debug/schema-updater NEXTEST_EXECUTION_MODE=process-per-test NEXTEST_LD_LIBRARY_PATH=/work/oxidecomputer/omicron/target/debug/build/aws-lc-sys-b2991dcfe0032b19/out:/work/oxidecomputer/omicron/target/debug/build/blake3-df20c6b18d3956e5/out:/work/oxidecomputer/omicron/target/debug/build/bzip2-sys-8b0f050bf31de6f5/out/lib:/work/oxidecomputer/omicron/target/debug/build/libgit2-sys-fc1bb2730a9a80d7/out/build:/work/oxidecomputer/omicron/target/debug/build/psm-e3b3fd8e545b20fd/out:/work/oxidecomputer/omicron/target/debug/build/ring-882060e0ad2a422c/out:/work/oxidecomputer/omicron/target/debug/build/tofino-d064271f908edbf6/out:/work/oxidecomputer/omicron/target/debug/build/zstd-sys-54b41a4eb78e9d96/out:/work/oxidecomputer/omicron/target/debug/deps:/work/oxidecomputer/omicron/target/debug:/home/build/.rustup/toolchains/1.91.1-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib:/home/build/.rustup/toolchains/1.91.1-x86_64-unknown-illumos/lib NEXTEST_PROFILE=ci NEXTEST_RUN_ID=ec1b80b1-e223-4a9a-80e8-f587c0ea1d10 NEXTEST_RUN_MODE=test NEXTEST_STATE_DIR=/tmp/nextest-state.EtaaBs NEXTEST_STRESS_CURRENT=none NEXTEST_STRESS_TOTAL=none NEXTEST_TEST_GLOBAL_SLOT=1 NEXTEST_TEST_GROUP=@global NEXTEST_TEST_GROUP_SLOT=none NEXTEST_TEST_NAME=integration_tests::volume_management::test_upstairs_notify_downstairs_client_stops NEXTEST_TEST_PHASE=run NEXTEST_TOTAL_ATTEMPTS=1 OUT_DIR=/work/oxidecomputer/omicron/target/debug/build/omicron-nexus-3b5642a0768e3b39/out PATH=/work/oxidecomputer/omicron/out/mgd/root/opt/oxide/mgd/bin:/work/oxidecomputer/omicron/out/dendrite-stub/bin:/work/oxidecomputer/omicron/out/clickhouse:/work/oxidecomputer/omicron/out/cockroachdb/bin:/home/build/.cargo/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/ooce/bin:/opt/ooce/sbin PWD=/work/oxidecomputer/omicron RUSTC_BOOTSTRAP=1 RUSTDOCFLAGS=--document-private-items -D warnings RUSTFLAGS=--cfg tokio_unstable -D warnings -C link-arg=-R/usr/platform/oxide/lib/amd64 RUSTUP_HOME=/home/build/.rustup RUSTUP_TOOLCHAIN=1.91.1-x86_64-unknown-illumos RUST_BACKTRACE=1 RUST_RECURSION_COUNT=1 SHLVL=1 SSL_CERT_DIR=/usr/ssl/certs SSL_CERT_FILE=/etc/ssl/cacert.pem TMPDIR=/var/tmp/omicron_tmp TZ=UTC USER=build _=/usr/bin/ptime","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:06.22804048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"cockroach pid: 3475","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:07.699821979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"cockroach listen URL: postgresql://root@[::1]:36930/omicron?sslmode=disable","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:07.69998362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"Running step start_clickhouse","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:07.792041213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Starting Clickhouse","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:07.79212995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"Running step start_internal_dns","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.300532209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"opening sled database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.300956812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"store","component":"internal_dns_server","path":"/var/tmp/omicron_tmp/.tmpyA2wFC"}
{"msg":"pruning trees for generations newer than 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.382354404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"store","component":"internal_dns_server"}
{"msg":"pruning trees for generations older than 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.382470074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"store","component":"internal_dns_server","keep":3}
{"msg":"DNS server bound to address","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.382703888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"dns","kind":"dns","component":"internal_dns_server","local_address":"[::1]:52138"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.383313279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:64048","component":"http","kind":"dns","component":"internal_dns_server","versions":"all starting from 2.0.0","path":"/config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.383433037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:64048","component":"http","kind":"dns","component":"internal_dns_server","versions":"from 1.0.0 to 2.0.0","path":"/config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.383477024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:64048","component":"http","kind":"dns","component":"internal_dns_server","versions":"all starting from 2.0.0","path":"/config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.383512013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:64048","component":"http","kind":"dns","component":"internal_dns_server","versions":"from 1.0.0 to 2.0.0","path":"/config","method":"PUT"}
{"msg":"listening","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.383542444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:64048","component":"http","kind":"dns","component":"internal_dns_server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.384380398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:64048","component":"http","kind":"dns","component":"internal_dns_server"}
{"msg":"Running step start_external_dns","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.384562911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"opening sled database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.384714081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"store","component":"external_dns_server","path":"/var/tmp/omicron_tmp/.tmpWLKUh8"}
{"msg":"pruning trees for generations newer than 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.393634848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"store","component":"external_dns_server"}
{"msg":"pruning trees for generations older than 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.393689316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"store","component":"external_dns_server","keep":3}
{"msg":"DNS server bound to address","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.393724757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"dns","kind":"dns","component":"external_dns_server","local_address":"[::1]:47749"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.393847701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:59802","component":"http","kind":"dns","component":"external_dns_server","versions":"all starting from 2.0.0","path":"/config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.393900396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:59802","component":"http","kind":"dns","component":"external_dns_server","versions":"from 1.0.0 to 2.0.0","path":"/config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.393936988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:59802","component":"http","kind":"dns","component":"external_dns_server","versions":"all starting from 2.0.0","path":"/config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.393967619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:59802","component":"http","kind":"dns","component":"external_dns_server","versions":"from 1.0.0 to 2.0.0","path":"/config","method":"PUT"}
{"msg":"listening","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39400333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:59802","component":"http","kind":"dns","component":"external_dns_server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.394881163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:59802","component":"http","kind":"dns","component":"external_dns_server"}
{"msg":"Running step start_nexus_internal","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.395299875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Starting Nexus (internal API)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.395353441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"setting up nexus server","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.395942722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398156448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Action"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398314072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AuditLog"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398394652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AnyActor"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398424741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AuthenticatedActor"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398468939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"BlueprintConfig"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398497756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Database"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398526654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"DnsConfig"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398553317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Fleet"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398584328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Inventory"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398622845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"IpPoolList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398654858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"VpcList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398683806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"ConsoleSessionList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398709447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"DeviceAuthRequestList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398737703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"QuiesceState"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398763013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloCertificateList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398791319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloGroupList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398816159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloIdentityProviderList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39884687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloUserList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398876308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloUserSessionList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39890265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloUserTokenList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.398971888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SubnetPoolList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399000375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"UpdateTrustRootList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399031757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"TargetReleaseConfig"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399057207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AlertClassList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399088279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"ScimClientBearerTokenList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399117397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"MulticastGroupList"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399300771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Project"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399353817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Disk"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39938067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Snapshot"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399409437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"ProjectImage"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399434938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AffinityGroup"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399463284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AntiAffinityGroup"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39949133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Instance"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399517371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"IpPool"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399560577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"InstanceNetworkInterface"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39958734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Vpc"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399618011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"VpcRouter"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399642941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"InternetGateway"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399674283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"InternetGatewayIpPool"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399754933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"InternetGatewayIpAddress"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.39978373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"RouterRoute"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399812538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"VpcSubnet"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399837637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"FloatingIp"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399867066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"ExternalSubnet"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399892797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Image"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399920512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloImage"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399948728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AddressLot"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.399973918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Blueprint"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400003657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"LoopbackAddress"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.40003004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Certificate"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400061312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"ConsoleSession"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400086512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"DeviceAuthRequest"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400116962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"DeviceAccessToken"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400147593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"PhysicalDisk"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400173575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Rack"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400201089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SshKey"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400225528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Silo"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400255097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloUser"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400280547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SiloGroup"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400308392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SupportBundle"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400333753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"IdentityProvider"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400361889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SamlIdentityProvider"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400389504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Sled"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400413892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"SubnetPool"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400445986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"TufRepo"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400471486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"TufArtifact"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400502328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"TufTrustRoot"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.40052868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Alert"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400556605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"AlertReceiver"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400581094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"WebhookSecret"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.40060922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Zpool"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400637486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"Service"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400664309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"UserBuiltin"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400693297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"ScimClientBearerToken"}
{"msg":"registering Oso class","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400718176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","class":"MulticastGroup"}
{"msg":"full Oso configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.400746342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","config":"#\n# Oso configuration for Omicron\n# This file is augmented by generated snippets.\n#\n\n#\n# ACTOR TYPES AND BASIC RULES\n#\n\n# `AnyActor` includes both authenticated and unauthenticated users.\nactor AnyActor {}\n\n# An `AuthenticatedActor` has an identity in the system.  All of our operations\n# today require that an actor be authenticated.\nactor AuthenticatedActor {}\n\n# For any resource, `actor` can perform action `action` on it if they're\n# authenticated and their role(s) give them the corresponding permission on that\n# resource.\nallow(actor: AnyActor, action: Action, resource) if\n    actor.authenticated and\n    has_permission(actor.authn_actor.unwrap(), action.to_perm(), resource);\n\n# Define role relationships\nhas_role(actor: AuthenticatedActor, role: String, resource: Resource)\n\tif resource.has_role(actor, role);\n\n#\n# ROLES AND PERMISSIONS IN THE FLEET/SILO/PROJECT HIERARCHY\n#\n# We define the following permissions for most resources in the system:\n#\n# - \"create_child\": required to create child resources (of any type)\n#\n# - \"list_children\": required to list child resources (of all types) of a\n#   resource\n#\n# - \"modify\": required to modify or delete a resource\n#\n# - \"read\": required to read a resource\n#\n# We define the following predefined roles for only a few high-level resources:\n# the Fleet (see below), Silo, Organization, and Project.  The specific roles\n# are oriented around intended use-cases:\n#\n# - \"admin\": has all permissions on the resource\n#\n# - \"collaborator\": has \"read\", \"list_children\", and \"create_child\", plus\n#   the \"admin\" role for child resources.  The idea is that if you're an\n#   Organization Collaborator, you have full control over the Projects within\n#   the Organization, but you cannot modify or delete the Organization itself.\n#\n# - \"viewer\": has \"read\" and \"list_children\" on a resource\n#\n# Below the Project level, permissions are granted via roles at the Project\n# level.  For example, for someone to be able to create, modify, or delete any\n# Instances, they must be granted project.collaborator, which means they can\n# create, modify, or delete _all_ resources in the Project.\n#\n# The complete set of predefined roles:\n#\n# - fleet.admin           (superuser for the whole system)\n# - fleet.collaborator    (can manage Silos)\n# - fleet.viewer          (can read most non-siloed resources in the system)\n# - silo.admin            (superuser for the silo)\n# - silo.collaborator     (can create and own Organizations; grants project.admin on all projects)\n# - silo.limited-collaborator (grants project.limited-collaborator on all projects)\n# - silo.viewer           (can read most resources within the Silo; grants project.viewer)\n# - organization.admin    (complete control over an organization)\n# - organization.collaborator (can manage Projects)\n# - organization.viewer   (can read most resources within the Organization)\n# - project.admin         (complete control over a Project)\n# - project.collaborator  (can manage all resources within the Project, including networking)\n# - project.limited-collaborator (can manage compute resources, but not networking resources)\n# - project.viewer        (can read most resources within the Project)\n#\n# Outside the Silo/Organization/Project hierarchy, we (currently) treat most\n# resources as nested under Fleet or else a synthetic resource (see below).  We\n# do not yet support role assignments on anything other than Fleet, Silo,\n# Organization, or Project.\n#\n\n# \"Fleet\" is a global singleton representing the whole system.  The name comes\n# from the idea described in RFD 24, but it's not quite right.  This probably\n# should be more like \"Region\" or \"AvailabilityZone\".  The precise boundaries\n# have not yet been figured out.\nresource Fleet {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"read\",\n\t    \"create_child\",\n\t];\n\n\troles = [\n\t    # Roles that can be attached by users\n\t    \"admin\",\n\t    \"collaborator\",\n\t    \"viewer\",\n\n\t    # Internal-only roles\n\t    \"external-authenticator\"\n\t];\n\n\t# Roles implied by other roles on this resource\n\t\"viewer\" if \"collaborator\";\n\t\"collaborator\" if \"admin\";\n\n\t# Permissions granted directly by roles on this resource\n\t\"list_children\" if \"viewer\";\n\t\"read\" if \"viewer\";\n\t\"create_child\" if \"collaborator\";\n\t\"modify\" if \"admin\";\n}\n\n# For fleets specifically, roles can be conferred by roles on the user's Silo.\n# Note that certain Actors may not ever have any roles assigned to them, like\n# SCIM Actors.\nhas_role(actor: AuthenticatedActor, role: String, _: Fleet) if\n\tsilo_role in actor.confers_fleet_role(role) and\n\thas_role(actor, silo_role, actor.silo.unwrap());\n\nresource Silo {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"read\",\n\t    \"create_child\",\n\t];\n\troles = [ \"admin\", \"collaborator\", \"limited-collaborator\", \"viewer\" ];\n\n\t# Roles implied by other roles on this resource\n\t\"viewer\" if \"limited-collaborator\";\n\t\"limited-collaborator\" if \"collaborator\";\n\t\"collaborator\" if \"admin\";\n\n\t# Permissions granted directly by roles on this resource\n\t\"list_children\" if \"viewer\";\n\t\"read\" if \"viewer\";\n\n\t\"create_child\" if \"collaborator\";\n\t\"modify\" if \"admin\";\n\n\t# Permissions implied by roles on this resource's parent (Fleet).  Fleet\n\t# privileges allow a user to see and potentially administer the Silo,\n\t# but they do not give anyone permission to look at anything inside the\n\t# Silo.  To achieve this, we use permission rules here.  (If we granted\n\t# Fleet administrators _roles_ on the Silo, then those would cascade\n\t# into the Silo as well.)\n\trelations = { parent_fleet: Fleet };\n\t\"read\" if \"viewer\" on \"parent_fleet\";\n\t\"modify\" if \"collaborator\" on \"parent_fleet\";\n\n\t# external authenticator has to create silo users\n\t\"list_children\" if \"external-authenticator\" on \"parent_fleet\";\n\t\"create_child\" if \"external-authenticator\" on \"parent_fleet\";\n}\n\nhas_relation(fleet: Fleet, \"parent_fleet\", silo: Silo)\n\tif silo.fleet = fleet;\n\n# As a special case, all authenticated users can read their own Silo.  That's\n# not quite the same as having the \"viewer\" role.  For example, they cannot list\n# Organizations in the Silo.\n#\n# One reason this is necessary is because if an unprivileged user tries to\n# create an Organization using \"POST /organizations\", they should get back a 403\n# (which implies they're able to see /organizations, which is essentially seeing\n# the Silo itself) rather than a 404.  This behavior isn't a hard constraint\n# (i.e., you could reasonably get a 404 for an API you're not allowed to call).\n# Nor is the implementation (i.e., we could special-case this endpoint somehow).\n# But granting this permission is the simplest way to keep this endpoint's\n# behavior consistent with the rest of the API.\n#\n# This rule is also used to determine if a user can list the identity providers\n# in the Silo (which they should be able to), since that's predicated on being\n# able to read the Silo.\n#\n# It's unclear what else would break if users couldn't see their own Silo.\nhas_permission(actor: AuthenticatedActor, \"read\", silo: Silo)\n\tif actor.is_user and silo in actor.silo;\n\nresource Project {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"read\",\n\t    \"create_child\",\n\t];\n\troles = [ \"admin\", \"collaborator\", \"limited-collaborator\", \"viewer\" ];\n\n\t# Roles implied by other roles on this resource\n\t# Role hierarchy: admin > collaborator > limited-collaborator > viewer\n\t#\n\t# The \"limited-collaborator\" role can create/modify non-networking\n\t# resources (instances, disks, etc.) but cannot create/modify networking\n\t# infrastructure (VPCs, subnets, routers, internet gateways).\n\t# See nexus/authz-macros for InProjectLimited vs InProjectFull.\n\t\"viewer\" if \"limited-collaborator\";\n\t\"limited-collaborator\" if \"collaborator\";\n\t\"collaborator\" if \"admin\";\n\n\t# Permissions granted directly by roles on this resource\n\t\"list_children\" if \"viewer\";\n\t\"read\" if \"viewer\";\n\t\"create_child\" if \"limited-collaborator\";\n\t\"modify\" if \"admin\";\n\n\t# Roles implied by roles on this resource's parent (Silo)\n\trelations = { parent_silo: Silo };\n\t\"admin\" if \"collaborator\" on \"parent_silo\";\n\t\"limited-collaborator\" if \"limited-collaborator\" on \"parent_silo\";\n\t\"viewer\" if \"viewer\" on \"parent_silo\";\n}\nhas_relation(silo: Silo, \"parent_silo\", project: Project)\n\tif project.silo = silo;\n\n#\n# GENERAL RESOURCES OUTSIDE THE SILO/PROJECT HIERARCHY\n#\n# Many resources use snippets of Polar generated by the `authz_resource!` Rust\n# macro.  Some resources require custom Polar code.  Those appear here.\n#\n\nresource Certificate {\n\tpermissions = [ \"read\", \"modify\" ];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Fleet-level and silo-level roles both grant privileges on certificates.\n\t\"read\" if \"admin\" on \"parent_silo\";\n\t\"modify\" if \"admin\" on \"parent_silo\";\n\t\"read\" if \"admin\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", certificate: Certificate)\n\tif certificate.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", certificate: Certificate)\n\tif certificate.silo.fleet = fleet;\n\nresource SiloUser {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"read\",\n\t    \"create_child\",\n\t];\n\n\t# Fleet and Silo administrators can manage a Silo's users.  This is one\n\t# of the only areas of Silo configuration that Fleet Administrators have\n\t# permissions on.\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\t\"list_children\" if \"read\" on \"parent_silo\";\n\t\"read\" if \"read\" on \"parent_silo\";\n\t\"modify\" if \"admin\" on \"parent_silo\";\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\t\"list_children\" if \"read\" on \"parent_fleet\";\n\t\"read\" if \"read\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", user: SiloUser)\n\tif user.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", user: SiloUser)\n\tif user.silo.fleet = fleet;\n\n# authenticated actors have all permissions on themselves\nhas_permission(actor: AuthenticatedActor, _perm: String, silo_user: SiloUser)\n    if actor.equals_silo_user(silo_user);\n\nhas_permission(actor: AuthenticatedActor, \"read\", silo_user: SiloUser)\n    if actor.is_user and silo_user.silo in actor.silo;\n\nresource SiloGroup {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"read\",\n\t    \"create_child\",\n\t];\n\n\trelations = { parent_silo: Silo };\n\t\"list_children\" if \"read\" on \"parent_silo\";\n\t\"read\" if \"read\" on \"parent_silo\";\n\t\"modify\" if \"admin\" on \"parent_silo\";\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n}\nhas_relation(silo: Silo, \"parent_silo\", group: SiloGroup)\n\tif group.silo = silo;\n\nresource SshKey {\n\tpermissions = [ \"read\", \"modify\" ];\n\trelations = { silo_user: SiloUser };\n\n\t\"read\" if \"read\" on \"silo_user\";\n}\n# We want to allow the user to modify the ssh key but disallow a SCIM IdP token\n# from doing the same.\nhas_permission(actor: AuthenticatedActor, \"modify\", ssh_key: SshKey)\n\tif actor.is_user and has_permission(actor, \"modify\", ssh_key.silo_user);\nhas_relation(user: SiloUser, \"silo_user\", ssh_key: SshKey)\n\tif ssh_key.silo_user = user;\n\nresource IdentityProvider {\n\tpermissions = [\n\t    \"read\",\n\t    \"modify\",\n\t    \"create_child\",\n\t    \"list_children\",\n\t];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Silo-level roles grant privileges on identity providers.\n\t\"read\" if \"viewer\" on \"parent_silo\";\n\t\"list_children\" if \"viewer\" on \"parent_silo\";\n\t\"modify\" if \"admin\" on \"parent_silo\";\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\n\t# Fleet-level roles also grant privileges on identity providers.\n\t\"read\" if \"viewer\" on \"parent_fleet\";\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", identity_provider: IdentityProvider)\n\tif identity_provider.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: IdentityProvider)\n\tif collection.silo.fleet = fleet;\n\nresource SamlIdentityProvider {\n\tpermissions = [\n\t    \"read\",\n\t    \"modify\",\n\t    \"create_child\",\n\t    \"list_children\",\n\t];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Silo-level roles grant privileges on identity providers.\n\t\"read\" if \"viewer\" on \"parent_silo\";\n\t\"list_children\" if \"viewer\" on \"parent_silo\";\n\t\"modify\" if \"admin\" on \"parent_silo\";\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\n\t# Fleet-level roles also grant privileges on identity providers.\n\t\"read\" if \"viewer\" on \"parent_fleet\";\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", saml_identity_provider: SamlIdentityProvider)\n\tif saml_identity_provider.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: SamlIdentityProvider)\n\tif collection.silo.fleet = fleet;\n\nresource ScimClientBearerToken {\n\tpermissions = [ \"read\", \"modify\" ];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n    # necessary to authenticate SCIM actors\n\t\"read\" if \"external-authenticator\" on \"parent_fleet\";\n\n\t# Silo-level roles grant privileges for SCIM client tokens.\n\t\"read\" if \"admin\" on \"parent_silo\";\n\t\"modify\" if \"admin\" on \"parent_silo\";\n\n\t# Fleet-level roles also grant privileges for SCIM client tokens.\n\t\"read\" if \"admin\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", scim_client_bearer_token: ScimClientBearerToken)\n\tif scim_client_bearer_token.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: ScimClientBearerToken)\n\tif collection.silo.fleet = fleet;\n\n\n#\n# SYNTHETIC RESOURCES OUTSIDE THE SILO HIERARCHY\n#\n# The resources here do not correspond to anything that appears explicitly in\n# the API or is stored in the database.  These are used either at the top level\n# of the API path (e.g., \"/v1/system/ip-pools\") or as an implementation detail of the system\n# (in the case of console sessions and \"Database\").  The policies are\n# either statically-defined in this file or driven by role assignments on the\n# Fleet.  None of these resources defines their own roles.\n#\n\n# Describes the quiesce state of a particular Nexus instance.\n#\n# These authz checks must not require the database.  We grant this directly to\n# callers of the internal API.\nresource QuiesceState {\n\tpermissions = [ \"read\", \"modify\" ];\n}\nhas_permission(USER_INTERNAL_API: AuthenticatedActor, \"read\", _q: QuiesceState);\nhas_permission(\n    USER_INTERNAL_API: AuthenticatedActor,\n    \"modify\",\n    _q: QuiesceState\n);\n\n# Describes the policy for reading and modifying DNS configuration\n# (both internal and external)\nresource DnsConfig {\n\tpermissions = [ \"read\", \"modify\" ];\n\trelations = { parent_fleet: Fleet };\n\t# \"external-authenticator\" requires these permissions because that's the\n\t# context that Nexus uses when creating and deleting Silos.  These\n\t# operations necessarily need to read and modify DNS configuration.\n\t\"read\" if \"external-authenticator\" on \"parent_fleet\";\n\t\"modify\" if \"external-authenticator\" on \"parent_fleet\";\n\t# \"admin\" on the parent fleet also gets these permissions, primarily for\n\t# the test suite.\n\t\"read\" if \"admin\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", dns_config: DnsConfig)\n\tif dns_config.fleet = fleet;\n\n# Describes the policy for accessing blueprints\nresource BlueprintConfig {\n\tpermissions = [\n\t    \"list_children\", # list blueprints\n\t    \"create_child\",  # create blueprint\n\t    \"read\",          # read the current target\n\t    \"modify\",        # change the current target\n\t];\n\n\trelations = { parent_fleet: Fleet };\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n\t\"read\" if \"viewer\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", list: BlueprintConfig)\n\tif list.fleet = fleet;\n\n# Describes the policy for accessing \"/v1/system/update/trust-roots\" in the API\nresource UpdateTrustRootList {\n\tpermissions = [ \"list_children\", \"create_child\" ];\n\trelations = { parent_fleet: Fleet };\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: UpdateTrustRootList)\n\tif collection.fleet = fleet;\n\n# Describes the policy for accessing blueprints\nresource TargetReleaseConfig {\n\tpermissions = [\n\t    \"read\",          # read the current target release\n\t    \"modify\",        # change the current target release\n\t];\n\n\trelations = { parent_fleet: Fleet };\n\t\"read\" if \"viewer\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", resource: TargetReleaseConfig)\n\tif resource.fleet = fleet;\n\n# Describes the policy for reading and modifying low-level inventory\nresource Inventory {\n\tpermissions = [ \"read\", \"modify\" ];\n\trelations = { parent_fleet: Fleet };\n\t\"read\" if \"viewer\" on \"parent_fleet\";\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", inventory: Inventory)\n\tif inventory.fleet = fleet;\n\n# Describes the policy for accessing \"/v1/system/ip-pools\" in the API\nresource IpPoolList {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"create_child\",\n\t];\n\n\t# Fleet Administrators can create or modify the IP Pools list.\n\trelations = { parent_fleet: Fleet };\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n\n\t# Fleet Viewers can list IP Pools\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", ip_pool_list: IpPoolList)\n\tif ip_pool_list.fleet = fleet;\n\n# Any authenticated user can create a child of a provided IP Pool.\n# This is necessary to use the pools when provisioning instances.\nhas_permission(actor: AuthenticatedActor, \"create_child\", ip_pool: IpPool)\n\tif actor.is_user and silo in actor.silo and silo.fleet = ip_pool.fleet;\n\n# Describes the policy for accessing \"/v1/multicast-groups\" in the API\n# Groups are created when the first instance joins and deleted when the last leaves.\nresource MulticastGroupList {\n\tpermissions = [ \"list_children\" ];\n\n\trelations = { parent_fleet: Fleet };\n\n\t# Fleet Viewers can list multicast groups\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", multicast_group_list: MulticastGroupList)\n\tif multicast_group_list.fleet = fleet;\n\n# Any authenticated user can list multicast groups in their fleet.\n# This enables silo users to discover groups for attaching instances,\n# without requiring the Fleet::Viewer role.\nhas_permission(actor: AuthenticatedActor, \"list_children\", multicast_group_list: MulticastGroupList)\n\tif actor.is_user and silo in actor.silo and silo.fleet = multicast_group_list.fleet;\n\n# MulticastGroup is a fleet-level discovery resource.\n# Join/leave authorization is gated by Instance::Modify, not the group itself.\nresource MulticastGroup {\n\tpermissions = [ \"read\", \"list_children\" ];\n\trelations = { parent_fleet: Fleet };\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", multicast_group: MulticastGroup)\n\tif multicast_group.fleet = fleet;\n\n# Any authenticated user can read multicast groups in their fleet\nhas_permission(actor: AuthenticatedActor, \"read\", multicast_group: MulticastGroup)\n\tif actor.is_user and silo in actor.silo and silo.fleet = multicast_group.fleet;\n\n# Any authenticated user can list members of multicast groups in their fleet\nhas_permission(actor: AuthenticatedActor, \"list_children\", multicast_group: MulticastGroup)\n\tif actor.is_user and silo in actor.silo and silo.fleet = multicast_group.fleet;\n\n# Describes the policy for reading and writing the audit log\nresource AuditLog {\n\tpermissions = [\n\t    \"list_children\", # retrieve audit log\n\t    \"create_child\",  # create audit log entry\n\t];\n\n\trelations = { parent_fleet: Fleet };\n\n\t# Fleet viewers can read the audit log\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n}\n\n# Any actor should be able to write to the audit log because we need to be able\n# to write to the log from any request, authenticated or not. Audit log writes\n# are always a byproduct of other operations: there are no endpoints that allow\n# the user to write to the log deliberately. Note we use AuthenticatedActor\n# because we don't really mean unauthenticated -- in the case of login\n# operations, we use the external authenticator actor that creates the session\n# to authorize the audit log write.\nhas_permission(_actor: AuthenticatedActor, \"create_child\", _audit_log: AuditLog);\n\nhas_relation(fleet: Fleet, \"parent_fleet\", audit_log: AuditLog)\n\tif audit_log.fleet = fleet;\n\n# Describes the policy for creating and managing web console sessions.\nresource ConsoleSessionList {\n\tpermissions = [ \"create_child\" ];\n\trelations = { parent_fleet: Fleet };\n\t\"create_child\" if \"external-authenticator\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: ConsoleSessionList)\n\tif collection.fleet = fleet;\n\n# Allow silo admins to delete and list user sessions\nresource SiloUserSessionList {\n    permissions = [ \"modify\", \"list_children\" ];\n    relations = { parent_silo: Silo };\n\n    # A silo admin can modify (e.g., delete) a user's sessions.\n    \"modify\" if \"admin\" on \"parent_silo\";\n\n    # A silo admin can list a user's sessions.\n    \"list_children\" if \"admin\" on \"parent_silo\";\n}\nhas_relation(silo: Silo, \"parent_silo\", authn_list: SiloUserSessionList)\n    if authn_list.silo_user.silo = silo;\n\n# give users 'modify' and 'list_children' on their own sessions\nhas_permission(actor: AuthenticatedActor, \"modify\", authn_list: SiloUserSessionList)\n    if actor.equals_silo_user(authn_list.silo_user);\nhas_permission(actor: AuthenticatedActor, \"list_children\", authn_list: SiloUserSessionList)\n    if actor.equals_silo_user(authn_list.silo_user);\n\n# Allow silo admins to delete and list user access tokens\nresource SiloUserTokenList {\n    permissions = [ \"modify\", \"list_children\" ];\n    relations = { parent_silo: Silo };\n\n    # A silo admin can modify (e.g., delete) a user's tokens.\n    \"modify\" if \"admin\" on \"parent_silo\";\n\n    # A silo admin can list a user's tokens.\n    \"list_children\" if \"admin\" on \"parent_silo\";\n}\nhas_relation(silo: Silo, \"parent_silo\", authn_list: SiloUserTokenList)\n    if authn_list.silo_user.silo = silo;\n\n# give users 'modify' and 'list_children' on their own tokens\nhas_permission(actor: AuthenticatedActor, \"modify\", authn_list: SiloUserTokenList)\n    if actor.equals_silo_user(authn_list.silo_user);\nhas_permission(actor: AuthenticatedActor, \"list_children\", authn_list: SiloUserTokenList)\n    if actor.equals_silo_user(authn_list.silo_user);\n\n# Describes the policy for creating and managing device authorization requests.\nresource DeviceAuthRequestList {\n\tpermissions = [ \"create_child\" ];\n\trelations = { parent_fleet: Fleet };\n\t\"create_child\" if \"external-authenticator\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: DeviceAuthRequestList)\n\tif collection.fleet = fleet;\n\n# Describes the policy for creating and managing Silo certificates\nresource SiloCertificateList {\n\tpermissions = [ \"list_children\", \"create_child\" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Both Fleet and Silo administrators can see and modify the Silo's\n\t# certificates.\n\t\"list_children\" if \"admin\" on \"parent_silo\";\n\t\"list_children\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", collection: SiloCertificateList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: SiloCertificateList)\n\tif collection.silo.fleet = fleet;\n\n# Describes the policy for creating and managing Silo identity providers\nresource SiloIdentityProviderList {\n\tpermissions = [ \"list_children\", \"create_child\" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Everyone who can read the Silo (which includes all the users in the\n\t# Silo) can see the identity providers in it.\n\t\"list_children\" if \"read\" on \"parent_silo\";\n\n\t# Fleet and Silo administrators can manage the Silo's identity provider\n\t# configuration.  This is one of the only areas of Silo configuration\n\t# that Fleet Administrators have permissions on.  This is also one of\n\t# the only cases where we need to look two levels up the hierarchy to\n\t# see if somebody has the right permission.  For most other things,\n\t# permissions cascade down the hierarchy so we only need to look at the\n\t# parent.\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", collection: SiloIdentityProviderList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: SiloIdentityProviderList)\n\tif collection.silo.fleet = fleet;\n\n# Describes the policy for creating and managing Silo users (mostly intended for\n# API-managed users)\nresource SiloUserList {\n\tpermissions = [ \"list_children\", \"create_child\" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Everyone who can read the Silo (which includes all the users in the\n\t# Silo) can see the users in it.\n\t\"list_children\" if \"read\" on \"parent_silo\";\n\n\t# Fleet and Silo administrators can manage the Silo's users.  This is\n\t# one of the only areas of Silo configuration that Fleet Administrators\n\t# have permissions on.  This is also one of the few cases (so far) where\n\t# we need to look two levels up the hierarchy to see if somebody has the\n\t# right permission.  For most other things, permissions cascade down the\n\t# hierarchy so we only need to look at the parent.\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\t\"list_children\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", collection: SiloUserList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: SiloUserList)\n\tif collection.silo.fleet = fleet;\n\n# Grant SCIM IdP actors the permissions they need on users.\nhas_permission(actor: AuthenticatedActor, \"read\", silo_user: SiloUser)\n    if actor.is_scim_idp and silo_user.silo in actor.silo;\nhas_permission(actor: AuthenticatedActor, \"create_child\", silo_user_list: SiloUserList)\n\tif actor.is_scim_idp and silo_user_list.silo in actor.silo;\nhas_permission(actor: AuthenticatedActor, \"modify\", silo_user: SiloUser)\n\tif actor.is_scim_idp and silo_user.silo in actor.silo;\nhas_permission(actor: AuthenticatedActor, \"list_children\", silo_user_list: SiloUserList)\n    if actor.is_scim_idp and silo_user_list.silo in actor.silo;\n\n# Describes the policy for creating and managing Silo groups (mostly intended\n# for API-managed groups)\nresource SiloGroupList {\n\tpermissions = [ \"list_children\", \"create_child\" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Everyone who can read the Silo (which includes all the groups in the\n\t# Silo) can see the groups in it.\n\t\"list_children\" if \"read\" on \"parent_silo\";\n\n\t# Fleet and Silo administrators can manage the Silo's groups.  This is\n\t# one of the only areas of Silo configuration that Fleet Administrators\n\t# have permissions on.  This is also one of the few cases (so far) where\n\t# we need to look two levels up the hierarchy to see if somebody has the\n\t# right permission.  For most other things, permissions cascade down the\n\t# hierarchy so we only need to look at the parent.\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\t\"list_children\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", collection: SiloGroupList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: SiloGroupList)\n\tif collection.silo.fleet = fleet;\n\n# Grant SCIM IdP actors the permissions they need on groups.\nhas_permission(actor: AuthenticatedActor, \"read\", silo_group: SiloGroup)\n    if actor.is_scim_idp and silo_group.silo in actor.silo;\nhas_permission(actor: AuthenticatedActor, \"create_child\", silo_group_list: SiloGroupList)\n\tif actor.is_scim_idp and silo_group_list.silo in actor.silo;\nhas_permission(actor: AuthenticatedActor, \"modify\", silo_group: SiloGroup)\n\tif actor.is_scim_idp and silo_group.silo in actor.silo;\nhas_permission(actor: AuthenticatedActor, \"list_children\", silo_group_list: SiloGroupList)\n    if actor.is_scim_idp and silo_group_list.silo in actor.silo;\n\n# These rules grants the external authenticator role the permissions it needs to\n# read silo users and modify their sessions.  This is necessary for login to\n# work.\nhas_permission(actor: AuthenticatedActor, \"read\", silo: Silo)\n\tif has_role(actor, \"external-authenticator\", silo.fleet);\nhas_permission(actor: AuthenticatedActor, \"read\", user: SiloUser)\n\tif has_role(actor, \"external-authenticator\", user.silo.fleet);\nhas_permission(actor: AuthenticatedActor, \"modify\", user: SiloUser)\n\tif has_role(actor, \"external-authenticator\", user.silo.fleet);\nhas_permission(actor: AuthenticatedActor, \"read\", group: SiloGroup)\n\tif has_role(actor, \"external-authenticator\", group.silo.fleet);\nhas_permission(actor: AuthenticatedActor, \"modify\", group: SiloGroup)\n\tif has_role(actor, \"external-authenticator\", group.silo.fleet);\n\nhas_permission(actor: AuthenticatedActor, \"read\", session: ConsoleSession)\n\tif has_role(actor, \"external-authenticator\", session.fleet);\nhas_permission(actor: AuthenticatedActor, \"modify\", session: ConsoleSession)\n\tif has_role(actor, \"external-authenticator\", session.fleet);\n\n# All authenticated users can read and delete device authn requests because\n# by necessity these operations happen before we've figured out what user (or\n# even Silo) the device auth is associated with.  Any user can claim a device\n# auth request with the right user code (that's how it works) -- it's the user\n# code and associated logic that prevents unauthorized access here.\nhas_permission(actor: AuthenticatedActor, \"read\", _device_auth: DeviceAuthRequest)\n    if actor.is_user;\nhas_permission(actor: AuthenticatedActor, \"modify\", _device_auth: DeviceAuthRequest)\n    if actor.is_user;\n\nhas_permission(actor: AuthenticatedActor, \"read\", device_token: DeviceAccessToken)\n\tif has_role(actor, \"external-authenticator\", device_token.fleet);\n\nhas_permission(actor: AuthenticatedActor, \"read\", identity_provider: IdentityProvider)\n\tif has_role(actor, \"external-authenticator\", identity_provider.silo.fleet);\n\nhas_permission(actor: AuthenticatedActor, \"read\", saml_identity_provider: SamlIdentityProvider)\n\tif has_role(actor, \"external-authenticator\", saml_identity_provider.silo.fleet);\n\n# Describes the policy for who can access the internal database.\nresource Database {\n\tpermissions = [\n\t    # \"query\" is required to perform any query against the database,\n\t    # whether a read or write query.  This is checked when an operation\n\t    # checks out a database connection from the connection pool.\n\t    #\n\t    # Any authenticated user gets this permission.  There's generally\n\t    # some other authz check involved in the database query.  For\n\t    # example, if you're querying the database to \"read\" a \"Project\", we\n\t    # should also be checking that.  So why do we do this at all?  It's\n\t    # a belt-and-suspenders measure so that if we somehow introduced an\n\t    # unauthenticated code path that hits the database, it cannot be\n\t    # used to DoS the database because we won't allow the operation to\n\t    # make the query.  (As long as the code path _is_ authenticated, we\n\t    # can use throttling mechanisms to prevent DoS.)\n\t    \"query\",\n\n\t    # \"modify\" is required to populate database data that's delivered\n\t    # with the system.  It should also be required for schema changes,\n\t    # when we support those.  This is separate from \"query\" so that we\n\t    # cannot accidentally invoke these code paths from API calls and\n\t    # other general functions.\n\t    \"modify\"\n\t];\n}\n\n# All authenticated users have the \"query\" permission on the database.\nhas_permission(_actor: AuthenticatedActor, \"query\", _resource: Database);\n\n# The \"db-init\" user is the only one with the \"modify\" permission.\nhas_permission(USER_DB_INIT: AuthenticatedActor, \"modify\", _resource: Database);\nhas_permission(USER_DB_INIT: AuthenticatedActor, \"create_child\", _resource: IpPoolList);\n# It also has \"admin\" on the internal silo to populate it with built-in resources.\n# TODO-completeness: actually limit to just internal silo and not all silos\nhas_role(USER_DB_INIT: AuthenticatedActor, \"admin\", _silo: Silo);\n\n# Allow the internal API admin permissions on all silos.\nhas_role(USER_INTERNAL_API: AuthenticatedActor, \"admin\", _silo: Silo);\n\nresource WebhookSecret {\n\tpermissions = [ \"read\", \"modify\" ];\n\trelations = { parent_alert_receiver: AlertReceiver };\n\n\t\"read\" if \"read\" on \"parent_alert_receiver\";\n\t\"modify\" if \"modify\" on \"parent_alert_receiver\";\n}\n\nhas_relation(rx: AlertReceiver, \"parent_alert_receiver\", secret: WebhookSecret)\n\tif secret.alert_receiver = rx;\n\nresource AlertClassList {\n\tpermissions = [ \"list_children\" ];\n\trelations = { parent_fleet: Fleet };\n\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n}\n\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: AlertClassList)\n\tif collection.fleet = fleet;\n\nresource ScimClientBearerTokenList {\n\tpermissions = [ \"create_child\", \"list_children\" ];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Silo-level roles grant privileges for SCIM client tokens.\n    # These are all admin because being able to create these tokens would allow\n    # a user to grant themselves admin by modifying group membership through SCIM calls\n\t\"create_child\" if \"admin\" on \"parent_silo\";\n\t\"list_children\" if \"admin\" on \"parent_silo\";\n\n    # Fleet-level roles also grant privileges for SCIM client tokens, for\n    # configuration before silo admins are present.\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n\t\"list_children\" if \"admin\" on \"parent_fleet\";\n}\nhas_relation(silo: Silo, \"parent_silo\", scim_client_bearer_token_list: ScimClientBearerTokenList)\n\tif scim_client_bearer_token_list.silo = silo;\nhas_relation(fleet: Fleet, \"parent_fleet\", collection: ScimClientBearerTokenList)\n\tif collection.silo.fleet = fleet;\n\n# VpcList is a synthetic resource for controlling VPC creation.\n# Unlike other project resources, VPC creation requires the full \"collaborator\"\n# role rather than \"limited-collaborator\", enforcing the networking restriction.\n# This allows organizations to restrict who can reconfigure the network topology\n# while still allowing users with limited-collaborator to work with compute\n# resources (instances, disks, etc.) within the existing network.\nresource VpcList {\n\tpermissions = [ \"list_children\", \"create_child\" ];\n\n\trelations = { containing_project: Project };\n\n\t\"list_children\" if \"read\" on \"containing_project\";\n\t\"create_child\" if \"collaborator\" on \"containing_project\";\n}\nhas_relation(project: Project, \"containing_project\", collection: VpcList)\n\tif collection.project = project;\n\n# Describes the policy for accessing \"/v1/system/subnet-pools\" in the API\nresource SubnetPoolList {\n\tpermissions = [\n\t    \"list_children\",\n\t    \"modify\",\n\t    \"create_child\",\n\t];\n\n\t# Fleet Administrators can create or modify the Subnet Pools list.\n\trelations = { parent_fleet: Fleet };\n\t\"modify\" if \"admin\" on \"parent_fleet\";\n\t\"create_child\" if \"admin\" on \"parent_fleet\";\n\n\t# Fleet Viewers can list External Subnet Pools\n\t\"list_children\" if \"viewer\" on \"parent_fleet\";\n}\nhas_relation(fleet: Fleet, \"parent_fleet\", subnet_pool_list: SubnetPoolList)\n\tif subnet_pool_list.fleet = fleet;\n\n\n\n                resource Disk {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: Disk)\n                        if child.project = parent;\n            \n\n                resource Snapshot {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: Snapshot)\n                        if child.project = parent;\n            \n\n                resource ProjectImage {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: ProjectImage)\n                        if child.project = parent;\n            \n\n                resource AffinityGroup {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: AffinityGroup)\n                        if child.project = parent;\n            \n\n                resource AntiAffinityGroup {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: AntiAffinityGroup)\n                        if child.project = parent;\n            \n\n                resource Instance {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: Instance)\n                        if child.project = parent;\n            \n\n                resource IpPool {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: IpPool)\n                    if child.fleet = fleet;\n            \n\n                resource InstanceNetworkInterface {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: Instance\n                    };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: InstanceNetworkInterface)\n                    if has_relation(project, \"containing_project\", child.instance);\n\n                has_relation(parent: Instance, \"parent\", child: InstanceNetworkInterface)\n                    if child.instance = parent;\n            \n\n                resource Vpc {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: Vpc)\n                        if child.project = parent;\n            \n\n                resource VpcRouter {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: Vpc\n                    };\n\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: VpcRouter)\n                    if has_relation(project, \"containing_project\", child.vpc);\n\n                has_relation(parent: Vpc, \"parent\", child: VpcRouter)\n                    if child.vpc = parent;\n            \n\n                resource InternetGateway {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: Vpc\n                    };\n\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: InternetGateway)\n                    if has_relation(project, \"containing_project\", child.vpc);\n\n                has_relation(parent: Vpc, \"parent\", child: InternetGateway)\n                    if child.vpc = parent;\n            \n\n                resource InternetGatewayIpPool {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: InternetGateway\n                    };\n\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: InternetGatewayIpPool)\n                    if has_relation(project, \"containing_project\", child.internet_gateway);\n\n                has_relation(parent: InternetGateway, \"parent\", child: InternetGatewayIpPool)\n                    if child.internet_gateway = parent;\n            \n\n                resource InternetGatewayIpAddress {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: InternetGateway\n                    };\n\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: InternetGatewayIpAddress)\n                    if has_relation(project, \"containing_project\", child.internet_gateway);\n\n                has_relation(parent: InternetGateway, \"parent\", child: InternetGatewayIpAddress)\n                    if child.internet_gateway = parent;\n            \n\n                resource RouterRoute {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: VpcRouter\n                    };\n\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: RouterRoute)\n                    if has_relation(project, \"containing_project\", child.vpc_router);\n\n                has_relation(parent: VpcRouter, \"parent\", child: RouterRoute)\n                    if child.vpc_router = parent;\n            \n\n                resource VpcSubnet {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = {\n                        containing_project: Project,\n                        parent: Vpc\n                    };\n\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"collaborator\" on \"containing_project\";\n                    \"create_child\" if \"collaborator\" on \"containing_project\";\n                }\n\n                has_relation(project: Project, \"containing_project\", child: VpcSubnet)\n                    if has_relation(project, \"containing_project\", child.vpc);\n\n                has_relation(parent: Vpc, \"parent\", child: VpcSubnet)\n                    if child.vpc = parent;\n            \n\n                resource FloatingIp {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: FloatingIp)\n                        if child.project = parent;\n            \n\n                resource ExternalSubnet {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_project: Project };\n                    \"list_children\" if \"viewer\" on \"containing_project\";\n                    \"read\" if \"viewer\" on \"containing_project\";\n                    \"modify\" if \"limited-collaborator\" on \"containing_project\";\n                    \"create_child\" if \"limited-collaborator\" on \"containing_project\";\n                }\n\n                has_relation(parent: Project, \"containing_project\", child: ExternalSubnet)\n                        if child.project = parent;\n            \n\n                resource Image {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_silo: Silo };\n                    \"list_children\" if \"viewer\" on \"containing_silo\";\n                    \"read\" if \"viewer\" on \"containing_silo\";\n                    \"modify\" if \"collaborator\" on \"containing_silo\";\n                    \"create_child\" if \"collaborator\" on \"containing_silo\";\n                }\n\n                has_relation(parent: Silo, \"containing_silo\", child: Image)\n                    if child.silo = parent;\n            \n\n                resource SiloImage {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n\n                    relations = { containing_silo: Silo };\n                    \"list_children\" if \"viewer\" on \"containing_silo\";\n                    \"read\" if \"viewer\" on \"containing_silo\";\n                    \"modify\" if \"collaborator\" on \"containing_silo\";\n                    \"create_child\" if \"collaborator\" on \"containing_silo\";\n                }\n\n                has_relation(parent: Silo, \"containing_silo\", child: SiloImage)\n                    if child.silo = parent;\n            \n\n                resource AddressLot {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: AddressLot)\n                    if child.fleet = fleet;\n            \n\n                resource Blueprint {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Blueprint)\n                    if child.fleet = fleet;\n            \n\n                resource LoopbackAddress {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: LoopbackAddress)\n                    if child.fleet = fleet;\n            \n\n\n                resource ConsoleSession {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: ConsoleSession)\n                    if child.fleet = fleet;\n            \n\n                resource DeviceAuthRequest {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: DeviceAuthRequest)\n                    if child.fleet = fleet;\n            \n\n                resource DeviceAccessToken {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: DeviceAccessToken)\n                    if child.fleet = fleet;\n            \n\n                resource PhysicalDisk {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: PhysicalDisk)\n                    if child.fleet = fleet;\n            \n\n                resource Rack {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Rack)\n                    if child.fleet = fleet;\n            \n\n\n\n\n\n                resource SupportBundle {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: SupportBundle)\n                    if child.fleet = fleet;\n            \n\n\n\n                resource Sled {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Sled)\n                    if child.fleet = fleet;\n            \n\n                resource SubnetPool {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: SubnetPool)\n                    if child.fleet = fleet;\n            \n\n                resource TufRepo {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: TufRepo)\n                    if child.fleet = fleet;\n            \n\n                resource TufArtifact {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: TufArtifact)\n                    if child.fleet = fleet;\n            \n\n                resource TufTrustRoot {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: TufTrustRoot)\n                    if child.fleet = fleet;\n            \n\n                resource Alert {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Alert)\n                    if child.fleet = fleet;\n            \n\n                resource AlertReceiver {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: AlertReceiver)\n                    if child.fleet = fleet;\n            \n\n\n                resource Zpool {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Zpool)\n                    if child.fleet = fleet;\n            \n\n                resource Service {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Service)\n                    if child.fleet = fleet;\n            \n\n                resource UserBuiltin {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: UserBuiltin)\n                    if child.fleet = fleet;\n            \n\n"}
{"msg":"Setting up resolver using DNS address: [::1]:52138","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.427533945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"new DNS resolver","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.427643192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","addresses":"[[::1]:52138]"}
{"msg":"Setting up qorb database pool from a single host","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:08.428712515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","url":"PostgresConfigWithUrl {\n    url_raw: \"postgresql://root@[::1]:36930/omicron?sslmode=disable\",\n    config: Config {\n        user: Some(\n            \"root\",\n        ),\n        password: None,\n        dbname: Some(\n            \"omicron\",\n        ),\n        options: None,\n        application_name: None,\n        ssl_mode: Disable,\n        host: [\n            Tcp(\n                \"::1\",\n            ),\n        ],\n        hostaddr: [],\n        port: [\n            36930,\n        ],\n        connect_timeout: None,\n        tcp_user_timeout: None,\n        keepalives: true,\n        keepalives_idle: 7200s,\n        keepalives_interval: None,\n        keepalives_retries: None,\n        target_session_attrs: Any,\n        channel_binding: Prefer,\n        load_balance_hosts: Disable,\n    },\n}"}
{"msg":"registered USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:08.431535051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"Database schema version is up to date","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:10.132602501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"desired_version":"234.0.0","found_version":"234.0.0","component":"datastore","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"No db_metadata_nexus records exist - skipping access check","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:10.163571558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"datastore","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","explanation":"This is expected during initial deployment or before migration","nexus_id":"913233fe-92a8-4635-9572-183f495429c4 (omicron_zone)"}
{"msg":"Datastore is ready for usage","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:10.163852877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"registered USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.377027279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"id":"39be4878-bb50-42ae-8b9a-fb2e33c1e4ac","component":"clickhouse-client","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.493776322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stop-request","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.49626263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stopped","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496340986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/crucible/0/upstairs/{upstairs_id}/repair/{repair_id}/progress","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496389071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/crucible/0/upstairs/{upstairs_id}/repair-finish","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496433099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/crucible/0/upstairs/{upstairs_id}/repair-start","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496465162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/disk/{disk_id}/remove-read-only-parent","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496505703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/disks/{disk_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496547065Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/metrics/collectors","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496577145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/metrics/collectors/{collector_id}/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496617315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/metrics/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496658536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/nat/ipv4/changeset/{from_gen}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496699788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/probes/{sled}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496729828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/refresh-vpc-routes","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.49677611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/sled-agents/{sled_id}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496820648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/sled-agents/{sled_id}","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496851299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/switch/{switch_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496890327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/v1/ping","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496930436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.496971358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/volume/{volume_id}/remove-read-only-parent","method":"POST"}
{"msg":"listening","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:10.497002249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.497711618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511169156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/bgtasks","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511323152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/bgtasks/activate","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511376347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/bgtasks/view/{bgtask_name}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511422288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/clickhouse/policy","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511454142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/clickhouse/policy","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511495704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/debug/fetch-omdb-binary","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511536976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/demo-saga","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511566816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/demo-saga/{demo_saga_id}/complete","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.51160933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/all","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511650121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/all/{blueprint_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511691814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/all/{blueprint_id}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.51175017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/import","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511816531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/regenerate","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511861821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/target","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511903093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/target","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511935428Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/blueprints/target/enabled","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.511978714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/reconfigurator-config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512046248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/reconfigurator-config","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512080546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/reconfigurator-config/{version}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512124473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/deployment/update-status","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512166246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.51220872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.51223871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512279942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512322336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.51236455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}/download","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512406854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}/download","method":"HEAD"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512437645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}/download/{file}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512481923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}/download/{file}","method":"HEAD"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512524989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/experimental/v1/system/support-bundles/{bundle_id}/index","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512567593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/instances/{instance_id}/migrate","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512597072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/mgs-updates","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512637793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/oximeter/read-policy","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512678834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/oximeter/read-policy","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512720647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/physical-disk/expunge","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512750647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/quiesce","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512796949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/quiesce","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512826648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/racks/{rack_id}/initialization-complete","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512856497Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/sagas","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512884432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/sagas/{saga_id}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512913009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/sleds/expunge","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512941235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/sleds/uninitialized","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512969531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/trust-quorum/config/{rack_id}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.512998088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/trust-quorum/lrtq-upgrade","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.513027266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/trust-quorum/remove/{sled}","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.513055572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4","versions":"all","path":"/v1/ping","method":"GET"}
{"msg":"listening","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:10.513083398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.513109129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:35350","component":"dropshot_lockstep","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"SEC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:10.513134259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sec_id":"913233fe-92a8-4635-9572-183f495429c4","component":"SEC","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:10.515014653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.515094742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.520419287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create built-in users","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.520631489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:10.52231865Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.522382126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:10.522897599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"starting MgsUpdateDriver","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:11.012126889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"MgsUpdateDriver","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"created 5 built-in users","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:11.027074887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.027861619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.027920887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.030465401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create built-in role assignments","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.030532474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.030561632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.030596431Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.030961706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.034259085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51832","req_id":"aac4c0ba-9264-41c0-9630-28c8da7c7668","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   20 06 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67    ............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.034758407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51832","req_id":"aac4c0ba-9264-41c0-9630-28c8da7c7668","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 8198,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 8198,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:11.035534268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51832","req_id":"aac4c0ba-9264-41c0-9630-28c8da7c7668","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.03558046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63547","req_id":"a8d1e4a4-4d6d-44b8-8a6e-3c9c0867a8fe","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   c5 aa 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.035630248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63547","req_id":"a8d1e4a4-4d6d-44b8-8a6e-3c9c0867a8fe","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50602,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50602,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:11.035885716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63547","req_id":"a8d1e4a4-4d6d-44b8-8a6e-3c9c0867a8fe","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.035914493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61343","req_id":"6447dcb5-8181-4a9e-b532-00e0187f1b3b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   f6 fa 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.035960645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61343","req_id":"6447dcb5-8181-4a9e-b532-00e0187f1b3b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 63226,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 63226,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:11.037507316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61343","req_id":"6447dcb5-8181-4a9e-b532-00e0187f1b3b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"created 4 built-in role assignments","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:11.247401023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.247653324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.247780116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.250135083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create built-in silos","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.250271063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.250355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.250439357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.25079308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"created 2 built-in silos","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:11.364210446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.364402447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.364453298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.365342995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.399474573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.399761814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.404244607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create built-in projects","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.40443224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.404831723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.40499107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.405944263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.419580305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.41966344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.419922795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.419988576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.420729598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.486556836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.486658528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.486695421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.487647552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.495204701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.498906113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.499080018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.499137222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.49917649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.499211449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.500406222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.511435798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.511548662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.511586156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.512285615Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.534516117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.538083642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.538151337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.538187719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.53854063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"created built-in services project","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:11.607707611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.607783332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.607822389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.610608984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create built-in VPCs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.610688241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.610719343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.610755154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.611145059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.615365541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.615433125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.615801206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.619113054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.619202842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.619244465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.619278763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.619641213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.623047087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.623115513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.623153458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.623550276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.640799825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.640939623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.640978891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.641487481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.64545964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.652696523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Project\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.652793656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.652823976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.652869376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.65290652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.652940568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.653345492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.658985714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.65905437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.659091033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.659516779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.663339892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.663421193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.663460772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.663845346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.694735296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.702012238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.702125403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.702161835Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.702505728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.716341347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.716469391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.716512557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.716549631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.717496421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.721535593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.721612686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.721651333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.722212808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.725187015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.725272826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.725312054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.725903448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.728383844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.754774348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.754900178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.754940378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.75581266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.799100122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.799237314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.799931253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.827724061Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.827890752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.828642355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.845897696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.84606598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.84611128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.846147191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.846694538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.857771899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.857866858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.857905063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.858591216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.890279682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.890372316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.890410903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.892101731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.955977042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.967964992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:11.973724721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.973803096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:11.974574659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.045450845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.050165639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.050403772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.050466737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.051165445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.067348736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.067421891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.067457592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.068623969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.080269137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.082967847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.083092304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.083808697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.119192091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.131424616Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.13152251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.131561027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.132176108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.232748574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.232876007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.232919293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.232956507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.233516609Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.247235115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.247309242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.247349272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.247947891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.252199044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.252266217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.252303091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.25293727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.269394936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.276710325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.27684353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.276886415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.277176171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create built-in VPC firewall rules","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.283697192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.283839064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.283891768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.294398015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.31399717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.314081267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.314825365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.327289891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.327393847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.327948178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.345319088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.345434828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.345476801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.345513965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.346150379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.35029965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.350362424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.350397915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.35122105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.364380265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.364446066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.364482388Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.365235904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.373832498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.383401954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Vpc\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.38355094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.383585047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.383636419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.383673112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.383707481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.384252112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.3889991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.389077826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.389121263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.390025148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.398570049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.398664797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.39875203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.399421239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.403517124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.415193575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.415290216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.415329034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.415631845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.427937164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.428042744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.428083505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.428117633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.428892852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.433080489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.43314126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.433177682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.434014915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.458505165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.481133817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.481757125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.482930756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.506453935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.532575645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.532885029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.532937694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.533481915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"created built-in VPC firewall rules","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:12.678749739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"attempting to create built-in VPC Subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.67882586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.678856301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.678930558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.679550539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.682988327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.683057234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.68351864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.685810362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.685871513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.686306167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.689223481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.689298981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.689338719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.6893741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.693016404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.69553215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.695599534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.695638812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.696022815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.702192117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.702282948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.702322216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.702708483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.705223438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.711582618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Vpc\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.711681705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.711712977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.711747746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.712128011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.720232116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.720386393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.720431572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.720467354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.72070707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.723436951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.723517301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.723556819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.724041813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.726584964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.726655894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.726696866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.727647784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.731015162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.738446331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.750538768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.755427217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.755508959Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.755546934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.756362053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.763315123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.76343951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.763482655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.763924012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.772003268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.772123878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.772167063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.772875751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.777589322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.785377761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.785507037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.785548009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.786045377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.903453148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.903553237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.904202957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.912088759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.912220801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.912269709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.91231058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.913159766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.918127964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.9182722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.918388782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.920383473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.955188679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:12.955392935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.955509987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.95707225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.964373902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:12.977111029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.041941888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.042099952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.042142266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.042177566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.04293983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.048291068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.048358582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.048396077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.048919156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.081653529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.081735832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.081772675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.08342623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.092140477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.126655827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.127336018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.127759369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.131750846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.141184963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.141257828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.141987477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.183923786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.184065087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.184110527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.184148813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.184688445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.22234807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.222485513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.222527025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.22306829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.231326832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.231412783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.231453294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.232560533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.267627849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.278698667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.285458372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.285573882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.285615274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.285650534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.286421705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.335365532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.335500861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.335542513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.336238746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.346511238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.346595646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.346634313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.347530593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.365296217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.372650463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) }","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.372746173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.372785611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.373712322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.381637622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.381714104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.382527119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"created built-in services vpc subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.411336747Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"created built-in services vpc","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.411700399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.411755078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.411793284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.41406229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"attempting to create silo users","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.414130566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.414221587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.414268029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.41478733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"created 2 silo users","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.50736141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"attempting to create silo user role assignments","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.507751394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.507924188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.508042392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.508900457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"created 2 silo user role assignments","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.528077414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.528170809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.528210067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.528776943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.534220073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.534285714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.534975544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.569560032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"IpPoolList","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.569652405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.569691202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.569724188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.598816267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.615270116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.622257493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"IpPoolList","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.622367081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.622406679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.622858717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.805223488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"IpPoolList","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.80535023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.805393937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.805430179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.80592408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.809456466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.811692136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"IpPoolList","action":"CreateChild","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.811766894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.811804659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.812253982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DataLoader","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001 (built_in_user), .. })"}
{"msg":"populate complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.816184949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.81636197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.816419575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.816459725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.816495325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.81692075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Running step start_gateway_switch0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.877237775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Starting Management Gateway","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.87730601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.914446454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.92059214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.920657109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.920692129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.921195899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating background tasks","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.931770531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.936450787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.936611927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.936663108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.936703028Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.937490963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.942708625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.942780218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.943288126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:13.943350931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.943543644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":1,"elapsed":"143.946µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.943691387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.943740415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.943774162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.943811175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.944484372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.945146988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.945207098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.945348589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:13.945396293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.945428938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":1,"elapsed":"54.207µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.945462755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.945494408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 06:56:13.945427004 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.945758653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.945808081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.94637162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.947264874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.947357198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.9474035Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.947442407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.947475763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.947999564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.957305546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.957432789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.957474592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.957985597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.95849604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.958555489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.958593614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.959043969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.959545334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.960014757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_executor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"Blueprint execution: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:13.960074806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_executor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no blueprint"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.960107591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_executor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no blueprint\"}","iteration":1,"elapsed":"53.918µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.960140937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.960174374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.960219012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.960256096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.960289973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.960807431Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.961497692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.961548202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.961584925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.961621187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.961655135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.962086121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.962600021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.962650391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.962687956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.962723346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.962757434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.965585751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.966682229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"reconfigurator config not yet loaded; doing nothing","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.966744042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.966856194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"Disabled\"","iteration":1,"elapsed":"163.936µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.966988668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.967042905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.967091832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.967129637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.967164136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.968022591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.968625799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"crdb_node_id_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"Blueprint execution: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:13.968698944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"crdb_node_id_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no blueprint"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.968730878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"crdb_node_id_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no blueprint\"}","iteration":1,"elapsed":"59.719µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.968764645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.968795877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.968830496Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.968866477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.968902799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.969344366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.969870892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.969921442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":1,"elapsed":"75.891µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.96995557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_rendezvous","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"Blueprint rendezvous: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:13.969987193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_rendezvous","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no blueprint"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970017934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_rendezvous","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no blueprint\"}","iteration":1,"elapsed":"46.212µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970066981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.970099376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970183964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":1,"elapsed":"96.051µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970235676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:13.970267991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970295154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":1,"elapsed":"35.62µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970327018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.970356156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.970390824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.970426936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970503378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.970812893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.971369037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.972299455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.972359765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.972905509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.973406784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.973460982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.973497344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.973912118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.974367312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.974485707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.974535636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.974995249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.975591534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.975738756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.975803334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.975843053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.975879696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.976346974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.976918349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.976971735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.97757379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":1,"elapsed":"656.394µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.977820561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.97799629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.978050177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.979072006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.979576037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"starting background task for service firewall rule propagation","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.979631056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.97966297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.979697669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.980128625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.980710771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.980768826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.980797673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.980832853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.980870127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.980903634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.981401322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.981842518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.98209485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.982149067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.982643058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.983160646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"saga_recovery","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"saga recovery pass starting","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.983304733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaQuiesceHandle","component":"NexusQuiesceHandle","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"listing candidate sagas for recovery","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.983353359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.983379942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000003a8a (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.983412487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.983859084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000003a8a (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.984447773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.984507071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.984544274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.985502868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.985907442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.985958203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.985993593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.986779834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.987604923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_garbage_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.9877343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_garbage_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.987785702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_garbage_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.993420283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_garbage_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.994613723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.994664785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.994700716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.995247051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.996900926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_finish","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.997040553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_finish","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.997077216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_finish","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.997416399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_finish","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.997855381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.997905471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.997942414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.997977985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.998012584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.998405203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.999092939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"reconfigurator config not yet loaded; doing nothing","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:13.999220883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.999256043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"reason\":\"reconfigurator config not yet loaded\",\"status\":\"disabled\"}","iteration":1,"elapsed":"121.692µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.999290542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"read_only_region_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:13.999320341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"read_only_region_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.999354849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"read_only_region_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:13.999723952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"read_only_region_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.000755551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.000819649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.000860289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.001296626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.001725247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.001983971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.002042698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.005847284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.006978541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.007034863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.007065403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.007093369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.007120422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.007393615Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.00745653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.00766286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.00770825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.00774353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.007779331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.007812878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.00946525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.010476037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.010530846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.010567469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.01108656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.011709447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:14.011762743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.011791039Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.011825768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.011861719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.011894935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.012405178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.014270894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.014337336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.014377466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.014415621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.014451493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.014929221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.015600956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":1,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:14.015662057Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.015692678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"Running step start_dendrite_switch0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.105336808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Starting Dendrite for switch0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.105616303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.117212865Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.117297283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.117344577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.117385879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.118820628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.123913363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:27866","req_id":"4124f5a5-849d-4813-a302-b6d44418ce45","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   19 d1 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   ............._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.124901075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:27866","req_id":"4124f5a5-849d-4813-a302-b6d44418ce45","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6609,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6609,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.125332381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:27866","req_id":"4124f5a5-849d-4813-a302-b6d44418ce45","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.125375437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:31707","req_id":"f3e57d70-cc36-4f58-895b-c627cd7628f4","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   c6 c8 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   ............._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.125425967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:31707","req_id":"f3e57d70-cc36-4f58-895b-c627cd7628f4","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50888,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50888,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.125697546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:31707","req_id":"f3e57d70-cc36-4f58-895b-c627cd7628f4","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.125727296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.125772586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.12581498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.125854749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.125887694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:14.128885749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.128960978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":1,"elapsed":"186.246992ms"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:14.129267487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.129325201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":1,"elapsed":"184.178865ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.129399018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:18469","req_id":"5395d7e8-5cc6-46ae-a7df-794491c6cfaa","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   a9 6a 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .j..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.129452354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:18469","req_id":"5395d7e8-5cc6-46ae-a7df-794491c6cfaa","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43370,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43370,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.129730697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:18469","req_id":"5395d7e8-5cc6-46ae-a7df-794491c6cfaa","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.129761778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50472","req_id":"e1d0053b-6f00-4dda-a887-6f43fb5509e0","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   b0 a3 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.129810605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50472","req_id":"e1d0053b-6f00-4dda-a887-6f43fb5509e0","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45219,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45219,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.130808036Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50472","req_id":"e1d0053b-6f00-4dda-a887-6f43fb5509e0","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.130861603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64469","req_id":"4f0627d7-2aa0-4b35-8067-f5c67ff6026f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   fb 5b 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .[..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.147691578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64469","req_id":"4f0627d7-2aa0-4b35-8067-f5c67ff6026f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 64347,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 64347,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.15457535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64469","req_id":"4f0627d7-2aa0-4b35-8067-f5c67ff6026f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.154634939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.154685409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.154729546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.154773764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":1,"elapsed":"129.101186ms"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.154827361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.154875647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":1,"elapsed":"130.238855ms"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.154925536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.154969152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.155009803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.155051947Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.155085924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:14.172813362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 06:56:13.945427004 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.173206173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T06:56:13.945427004Z\",\"pruned\":[]}","iteration":1,"elapsed":"227.67436ms"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.173645335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.175372966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.198945523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":1,"elapsed":"239.340129ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.203258349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53423","req_id":"2f084247-b208-4554-af69-111170e60b60","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   c1 d7 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.203340822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53423","req_id":"2f084247-b208-4554-af69-111170e60b60","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 49623,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 49623,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.203769424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53423","req_id":"2f084247-b208-4554-af69-111170e60b60","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.203810094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:18521","req_id":"47505990-0717-4ebe-a085-b8918f28eda1","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   13 c1 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.20385822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:18521","req_id":"47505990-0717-4ebe-a085-b8918f28eda1","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 5057,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 5057,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.204117906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:18521","req_id":"47505990-0717-4ebe-a085-b8918f28eda1","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.204601186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.204660794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.2046988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":1,"elapsed":"247.417211ms"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.204866342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:14.20492068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.204994417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":1,"elapsed":"246.452715ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.271367299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.29130695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.291486958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.291543651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.291980598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.292904754Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.296731274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.29685515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.296896973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.297119815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.29780671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.3032802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.303408154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.303447633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.303663421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Running step start_mgd_switch0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.309480675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Starting mgd for switch0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.309549712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.337355717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.342319826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:14.342438191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.342478441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.342732045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"mgd port is 55910","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.581720977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471}
{"msg":"Running step record_switch_dns","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.58340417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"Recording DNS for the switch zones","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.583919153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"switch_location":"switch0","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"Running step start_sled1","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:14.584120222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}
{"msg":"setting up sled agent server","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:14.584788038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"created simulated sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:17.991475771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","sim_mode":"Explicit"}
{"msg":"registered sled b6d65341-167c-41df-9b5c-41cded99c229 storage","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:17.993592044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"omicron_sled_agent::sim::SimulatedUpstairs"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:18.008144005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.009308989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.015947303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.053821955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.250197594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:18.251204674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.252024502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.260601707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.280668221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":1,"elapsed":"4.283787786s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.304543118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.41062702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:18.411894186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.412619467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.421525184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.445192229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.623692836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:18.624833211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.625553802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.659838284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.6668151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.874812029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:18.912674638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.921042647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:18.994004025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:19.078518738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.079549956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.093704488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:19.124660058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.125547531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.132562163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.160517174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.330816071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:19.331891307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:19.332617178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:19.333340395Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.334020435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.33872031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"listed in-progress sagas","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:19.389708116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","count":0}
{"msg":"saga recovery pass finished","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:19.392673966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaQuiesceHandle","component":"NexusQuiesceHandle","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","blueprint_id":"None","generation":"1"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.394309675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"saga_recovery","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"last_pass\":{\"Success\":{\"nfailed\":0,\"nfound\":0,\"nrecovered\":0,\"nremoved\":0,\"nskipped\":0}},\"ntotal_failures\":0,\"ntotal_finished\":0,\"ntotal_recovered\":0,\"ntotal_sec_errors_bad_state\":0,\"ntotal_sec_errors_missing\":0,\"ntotal_started\":0,\"recent_failures\":[],\"recent_recoveries\":[]}","iteration":1,"elapsed":"5.410643094s"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.411074952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:19.41247346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.414239748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":1,"elapsed":"5.432104768s"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:19.419617939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:40564"}
{"msg":"No ledger in /var/tmp/omicron_tmp/.tmpjVbLmb/artifact-config.json","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:19.593457639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"Failed to read ledger: Not found in storage","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.596921278Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","path":"/var/tmp/omicron_tmp/.tmpjVbLmb/artifact-config.json"}
{"msg":"No ledger in /var/tmp/omicron_tmp/.tmpZywEx5/artifact-config.json","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:19.599869393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"Failed to read ledger: Not found in storage","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:19.601585381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","path":"/var/tmp/omicron_tmp/.tmpZywEx5/artifact-config.json"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:22.830744226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:61760","component":"dropshot (Repo Depot)","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/artifact/sha256/{sha256}","method":"GET"}
{"msg":"listening","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:22.833260243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:61760","component":"dropshot (Repo Depot)","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:22.997872357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:61760","component":"dropshot (Repo Depot)","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.2439715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/artifacts","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.255460929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/artifacts/{sha256}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.256619801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/artifacts/{sha256}/copy-from-depot","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.257264391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/artifacts-config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.257869002Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/artifacts-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.258575234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/bootstore/status","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.259268611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 2.0.0","path":"/chicken-switch/destroy-orphaned-datasets","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.259901117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 3.0.0","path":"/chicken-switch/destroy-orphaned-datasets","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.260525658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 3.0.0","path":"/debug/switch-zone-policy","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.261139206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 3.0.0","path":"/debug/switch-zone-policy","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.261750861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 21.0.0","path":"/disks/{disk_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.262456423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/eip-gateways","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.263051404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 24.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.263651566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 22.0.0 to 24.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.264248041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 16.0.0 to 22.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.264845197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 14.0.0 to 16.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.265444337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 12.0.0 to 14.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.26604011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 11.0.0 to 12.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.266638308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 10.0.0 to 11.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.267233881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 4.0.0 to 10.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.267830817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 4.0.0","path":"/inventory","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.268429315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 17.0.0","path":"/local-storage","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.269035599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 17.0.0","path":"/local-storage","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.269643316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 9.0.0 to 17.0.0","path":"/local-storage/{zpool_id}/{dataset_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.270265833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 9.0.0 to 17.0.0","path":"/local-storage/{zpool_id}/{dataset_id}","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.271037165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 20.0.0 to 23.0.0","path":"/network-bootstore-config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.271652697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 20.0.0","path":"/network-bootstore-config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.272265214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 20.0.0","path":"/network-bootstore-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.272878021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 20.0.0","path":"/network-bootstore-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.273491389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 14.0.0","path":"/omicron-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.27409591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 11.0.0 to 14.0.0","path":"/omicron-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.274698797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 10.0.0 to 11.0.0","path":"/omicron-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.275599756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 4.0.0 to 10.0.0","path":"/omicron-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.276317121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 4.0.0","path":"/omicron-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.276929247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 10.0.0","path":"/probes","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.277530621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 6.0.0 to 10.0.0","path":"/probes","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.278124491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 19.0.0","path":"/rot/{rot}/attest","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.278729252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 19.0.0","path":"/rot/{rot}/certificate-chain","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.279346207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 19.0.0","path":"/rot/{rot}/measurement-log","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.279960347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/sled-identifiers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.280550038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 8.0.0","path":"/sled-role","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.281148827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/sleds","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.281727707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/dladm-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.282432707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/health-check","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.283316783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/ipadm-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.284424433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/logs/download/{zone}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.285044384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/logs/zones","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.285636109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/nvmeadm-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.286226271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/pargs-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.2870765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/pfiles-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.287736932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/pstack-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.288328907Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/zfs-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.288914941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/zoneadm-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.289505915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support/zpool-info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.290098071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.290706339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.29133628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.291963967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.292597656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download","method":"HEAD"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.293510398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download/{file}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.294162724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download/{file}","method":"HEAD"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.294802484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.295435953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/index","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.296066184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/index","method":"HEAD"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.296697879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.297330585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 20.0.0","path":"/switch-ports","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.298218689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 20.0.0","path":"/switch-ports","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.298844462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/commit","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.299562498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/configuration","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.30018277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/coordinator-status","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.30080209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 15.0.0","path":"/trust-quorum/network-config","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.301418354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 15.0.0","path":"/trust-quorum/network-config","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.302034458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/prepare-and-commit","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.302761893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/proxy/commit","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.303382205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/proxy/prepare-and-commit","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.304005372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/proxy/status","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.304619933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 15.0.0","path":"/trust-quorum/status","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.305226928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 13.0.0","path":"/trust-quorum/upgrade","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.305835717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/v2p","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.306414908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/v2p","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.306987305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/v2p","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.307560433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.308149302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 18.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.308753843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 17.0.0 to 18.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.309354606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 11.0.0 to 17.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.309959437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 10.0.0 to 11.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.310562505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 9.0.0 to 10.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.31115927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"from 7.0.0 to 9.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.311757258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 7.0.0","path":"/vmms/{propolis_id}","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.312373482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 18.0.0","path":"/vmms/{propolis_id}/attached-subnets","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.312993453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 18.0.0","path":"/vmms/{propolis_id}/attached-subnets","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.313616631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 18.0.0","path":"/vmms/{propolis_id}/attached-subnets","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.314235851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 18.0.0","path":"/vmms/{propolis_id}/attached-subnets/{subnet}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.314863968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/disks/{disk_id}/snapshot","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.31547393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/external-ip","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.316072989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/external-ip","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.316669965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 7.0.0","path":"/vmms/{propolis_id}/multicast-group","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.317290377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 7.0.0","path":"/vmms/{propolis_id}/multicast-group","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.317917944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/poke","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.318514408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/poke-single-step","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.319116845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/sim-migration-source","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.319727397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/state","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.32031784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vmms/{propolis_id}/state","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.331602783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all starting from 10.0.0","path":"/vpc/{vpc_id}/firewall/rules","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.332924728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all ending with 10.0.0","path":"/vpc/{vpc_id}/firewall/rules","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.33375139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vpc-routes","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.334515738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/vpc-routes","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.335703728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.336656159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundle-cleanup","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.337510306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundle-cleanup/context","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.338279824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundle-cleanup/context","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.338888152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundle-cleanup/utilization","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.33966237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundles","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.340834107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundles/{zone_name}","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.341615649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundles/{zone_name}/{bundle_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.342420468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","versions":"all","path":"/zones/bundles/{zone_name}/{bundle_id}","method":"GET"}
{"msg":"listening","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:23.343181479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.343745449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:23.344446141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"b34a5ec3-2c70-47c5-b391-ffa2b83c0316","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.345651435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"b34a5ec3-2c70-47c5-b391-ffa2b83c0316","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.346533647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.34698313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:23.363840089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"b34a5ec3-2c70-47c5-b391-ffa2b83c0316","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.364951015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"b34a5ec3-2c70-47c5-b391-ffa2b83c0316","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:23.379708365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"b34a5ec3-2c70-47c5-b391-ffa2b83c0316","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:24.393356373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:37675"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.461246889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0ac76ecc-586f-4265-a0b6-bc4a653681cb","remote_addr":"127.0.0.1:37675","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.462164081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0ac76ecc-586f-4265-a0b6-bc4a653681cb","remote_addr":"127.0.0.1:37675","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:24.49313925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.549900563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.55083072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.580792276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"no more instances in need of reincarnation","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.593729421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reincarnation_reason":"instance failed","total_found":0}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.598524714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.599615962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.615092149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.648457325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.650120919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.667926873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.697648973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49425","req_id":"7ab3fa31-ef54-4c1a-af6e-9b9f5d746b13","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   a1 e3 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.740885855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49425","req_id":"7ab3fa31-ef54-4c1a-af6e-9b9f5d746b13","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 41443,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 41443,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:24.761696918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49425","req_id":"7ab3fa31-ef54-4c1a-af6e-9b9f5d746b13","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.762321048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52134","req_id":"7aa125d7-8290-4537-98e5-585284d38e8e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   44 2a 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   D*..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.763241125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52134","req_id":"7aa125d7-8290-4537-98e5-585284d38e8e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 17450,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 17450,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:24.769146263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52134","req_id":"7aa125d7-8290-4537-98e5-585284d38e8e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.769687187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59325","req_id":"713711f3-6502-44e1-b985-32f4d3df8e22","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   e0 6a 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .j..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:24.770588377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59325","req_id":"713711f3-6502-44e1-b985-32f4d3df8e22","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 57450,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 57450,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:24.775955176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59325","req_id":"713711f3-6502-44e1-b985-32f4d3df8e22","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:24.99109478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:25.007731162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:25.011650776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.013207749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.022994576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.05535154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.173109086Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:25.17411826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:25.175142434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:25.175883506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.176569108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.181109205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.250776029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.317780455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:25.318737045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.319453378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.342748053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:25.59194082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.596320849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":1,"elapsed":"11.590598804s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:25.816796982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.075530669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.111858679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.277157497Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.794105056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.795207315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.796051612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.796847373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.804106901Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.816827114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.817908462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":true}","iteration":1,"elapsed":"12.854025202s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.819100369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.820101868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.821530836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.822326827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.823025445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.827691211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.864672629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Dependency"}
{"msg":"blueprint planning disabled, doing nothing","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.87572497Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.899776619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"Disabled\"","iteration":2,"elapsed":"35.395128ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:26.929733786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Dependency"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.958998168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.960840547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:26.965881619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.078176846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.099543963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:27.132416441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.133356778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.139694125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.146787703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":1,"elapsed":"13.184722704s"}
{"msg":"inventory_prune_one: nothing eligible for removal (too few)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.236270453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","candidates":"[]"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:27.237057637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:27.256302659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45706","req_id":"254cd810-0f88-494a-b86b-a1850adedad2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   01 ca 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.321629453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45706","req_id":"254cd810-0f88-494a-b86b-a1850adedad2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 458,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 458,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:27.327167613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45706","req_id":"254cd810-0f88-494a-b86b-a1850adedad2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"inventory collection failed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:27.545784764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"failed to collect inventory: looking up MGS addresses: proto error: no records found for Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:27.547021761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to collect inventory: looking up MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":1,"elapsed":"13.585593908s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:28.373609993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:28.400960343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:28.433145155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:28.520724354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":1,"elapsed":"14.549859168s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:28.676558023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_finish","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"finish_invoked_ok\":[]}","iteration":1,"elapsed":"14.678999601s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:28.710861382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_garbage_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"garbage_collect_requested\":[]}","iteration":1,"elapsed":"14.722789882s"}
{"msg":"failed to read target blueprint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:28.715197364Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: no target blueprint set"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:28.71567377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read target blueprint: Internal Error: no target blueprint set\"}","iteration":1,"elapsed":"14.747839573s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:29.137605932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:29.138401812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:29.138729724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:29.139017154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:29.139853566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:29.145940004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:29.868183769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"requests_completed_ok\":[],\"requests_created_ok\":[],\"start_invoked_ok\":[]}","iteration":1,"elapsed":"15.881895601s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:30.034609835Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"read_only_region_replacement_start","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"requests_created_ok\":[]}","iteration":1,"elapsed":"16.035004347s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:30.636214642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"b34a5ec3-2c70-47c5-b391-ffa2b83c0316","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":7321760,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:30.685464526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:30.81055744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:30.811611343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:30.812338518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:30.813061143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:30.813859498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:30.820787137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:31.028943583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:31.215533857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:31.216582309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:31.217331698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:31.236796246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:32.271639369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:32.346882619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:32.84406329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:32.845238494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":1,"elapsed":"18.900059683s"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:32.910330382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:32.911560475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":1,"elapsed":"18.973432457s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:32.996318982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.156313194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:33.157396967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.158127828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.169517188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:33.203999042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:33.204840183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.205549451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.216000087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.660408519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":1,"elapsed":"19.65213144s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.66176849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:33.66264456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:33.663537163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:33.664814309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.665792272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.689316032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:33.866613429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.116185971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.117373259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.118480808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.140886628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.1843651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.405893324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.442572141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.443888355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.452622181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:34.53241707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.533495422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":1,"elapsed":"20.557210081s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.535326178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"requests_completed_ok\":[],\"requests_created_ok\":[],\"start_invoked_ok\":[]}","iteration":1,"elapsed":"20.561145026s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.537550275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.538559479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.548223974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.562824402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.65023664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.666997809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.66839312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.688576135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.717834104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.719439532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.720211425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.720912097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.740023764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.786144714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.786991666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":1,"elapsed":"20.77089223s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.792641647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.793720299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:34.794479397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.795172093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:34.841788807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:35.175362873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:35.400655783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:35.486964279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:35.490483098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:35.518661322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:35.621351223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_replacement_driver","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"drive_invoked_ok\":[],\"errors\":[],\"finish_invoked_ok\":[]}","iteration":1,"elapsed":"21.646573916s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:35.947118914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.130630423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.153182994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.154810126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.159865787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.339247614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.542647603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.543992665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.545359229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.550276645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.638596716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53110","req_id":"7ec722e8-3b61-4ad8-aef0-46e745acfd12","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   f2 f2 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.640483744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53110","req_id":"7ec722e8-3b61-4ad8-aef0-46e745acfd12","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62194,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62194,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:36.647194332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53110","req_id":"7ec722e8-3b61-4ad8-aef0-46e745acfd12","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.647750797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50069","req_id":"f328ecfe-d7c0-41a0-acdd-b9b977160876","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   d8 19 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.648933526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50069","req_id":"f328ecfe-d7c0-41a0-acdd-b9b977160876","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55321,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55321,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:36.655578684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50069","req_id":"f328ecfe-d7c0-41a0-acdd-b9b977160876","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.656284656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54514","req_id":"d5c12496-9abd-4036-9d0b-07c6a5b8fb41","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   80 64 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .d..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.65736921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54514","req_id":"d5c12496-9abd-4036-9d0b-07c6a5b8fb41","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32868,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32868,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:36.664571254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54514","req_id":"d5c12496-9abd-4036-9d0b-07c6a5b8fb41","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.713912931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.715174967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:36.715929415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.716618464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.747469487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:36.786680783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.134562876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.135661327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.136384924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.142427055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.271032606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.33154556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":2,"elapsed":"10.513993987s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.375046416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.438190165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.439842797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.444732117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.46430459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.614804986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.624081931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.624838644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":1,"elapsed":"23.62366374s"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:37.627106678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.627846498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":1,"elapsed":"23.613315607s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.630436852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.677560704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:37.6784599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.679460177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.712029302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:37.737652451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.033136176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:38.034731093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.036220321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.041467342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.381591537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Err(reqwest::Error { kind: Request, url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", source: TimedOut })"}
{"msg":"failed to contact nexus, will retry in 105.339691ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:38.382589589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Communication Error: error sending request for url (http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229)"}
{"msg":"request handling cancelled (client disconnected)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:38.487282879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0ac76ecc-586f-4265-a0b6-bc4a653681cb","remote_addr":"127.0.0.1:37675","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":14026067}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.628186285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.700721315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.752087322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":2,"elapsed":"9.614524667s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:38.796987287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":2,"elapsed":"5.136045919s"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:38.826730589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:60153"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:39.619627679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"3b27283c-cabb-4c31-bab8-358606d519c3","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:39.620886008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"3b27283c-cabb-4c31-bab8-358606d519c3","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:39.704676402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.096275189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:40.097527006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.098263969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.106968427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:40.156066209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.1639663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.421293503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:40.422527013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.424604749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.449167833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.457259464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:40.53120191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:41.272755196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:41.293240421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":3,"elapsed":"3.917952806s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:41.620835462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:41.621807482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:41.642526753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:42.082413614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"TargetReleaseConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:42.083385263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:42.084134562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.084820153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.100562228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:42.580673028Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.58167578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":1,"elapsed":"28.633096154s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.642402288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:42.643337174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:42.64408543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:42.644943054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.645649196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.658809333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:42.968042536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.172105662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"TargetReleaseConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:43.173176158Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.174525648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.227529108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:43.321970255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:43.324405662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.325796515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.331643758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.610715918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.951822604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:43.953390868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.954655951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:43.970821337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.285848998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.288383722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.290758729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.312407435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.3316085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.333483464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.356676667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.400529432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.472617776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.496748902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.498198471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":2,"elapsed":"408.091µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.499377092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:44.500566394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.501381533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":2,"elapsed":"179.817µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.502266671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.52529116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.526400813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.527789431Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.528776371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.535234899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.541535442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.542731608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.543706243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.544695719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.545725243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.576521808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.581986261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:44.583004874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.583952406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":2,"elapsed":"681.123µs"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.640493631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.642335579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.643425985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.654595348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"No bundles to collect","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:44.662776006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.665508934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"support_bundle_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"cleanup_err\":null,\"cleanup_report\":{\"db_destroying_bundles_removed\":0,\"db_failing_bundles_updated\":0,\"sled_bundles_delete_failed\":0,\"sled_bundles_deleted_not_found\":0,\"sled_bundles_deleted_ok\":0},\"collection_err\":null,\"collection_report\":null}","iteration":1,"elapsed":"30.695796115s"}
{"msg":"request completed after handler was already cancelled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:44.669726139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0ac76ecc-586f-4265-a0b6-bc4a653681cb","remote_addr":"127.0.0.1:37675","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","response_code":204}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.675326051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Signaled"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.676300126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.677937929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.678943806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.679896088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.680708141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.696220499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"19d0d9ac-54de-4ef3-ba9f-6af140c8c324","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.730160397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"19d0d9ac-54de-4ef3-ba9f-6af140c8c324","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.736912487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.746534096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.747400988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.760935419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.7627083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.768319324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.769188059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.792988628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:44.953019262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"19d0d9ac-54de-4ef3-ba9f-6af140c8c324","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.954235458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"19d0d9ac-54de-4ef3-ba9f-6af140c8c324","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:44.962182742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"19d0d9ac-54de-4ef3-ba9f-6af140c8c324","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:45.156849776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:45.163837705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:45.164668655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.165231453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":1,"elapsed":"31.153372509s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.165918418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:45.166779518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.167867138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.205119855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"tuf_repo_prune decision","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:45.215450232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"TufRepoPrunerDetails { nkeep_recent_releases: 3, nkeep_recent_uploads: 2, repos_keep_target_release: {}, repos_keep_recent_uploads: {}, repo_prune: None, other_repos_eligible_to_prune: {}, warnings: [] }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.216324368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"nkeep_recent_releases\":3,\"nkeep_recent_uploads\":2,\"other_repos_eligible_to_prune\":[],\"repo_prune\":null,\"repos_keep_recent_uploads\":[],\"repos_keep_target_release\":[],\"status\":\"enabled\",\"warnings\":[]}","iteration":2,"elapsed":"18.28668527s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.300816997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.335987489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":4,"elapsed":"2.693613688s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:45.388123495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7991","req_id":"180bea09-f024-4b3a-a7ff-59baf85b0ea0","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   04 4e 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .N..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.405613171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7991","req_id":"180bea09-f024-4b3a-a7ff-59baf85b0ea0","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 1102,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 1102,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:45.413117014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7991","req_id":"180bea09-f024-4b3a-a7ff-59baf85b0ea0","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:45.450056399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.451070353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":2,"elapsed":"716.983588ms"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:45.607507901Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.696727729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":2,"elapsed":"1.223728184s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:45.751601733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45398","req_id":"12d41ff8-844e-46eb-aab6-9c59eff6bb28","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   61 58 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   aX..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:45.752827608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45398","req_id":"12d41ff8-844e-46eb-aab6-9c59eff6bb28","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 24920,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 24920,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:45.774065257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45398","req_id":"12d41ff8-844e-46eb-aab6-9c59eff6bb28","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:46.087013869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.0933384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.210661594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:46.211735557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.212446128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.218076941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.227709713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.445618768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:46.4469992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.447735552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.4649595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:46.478767705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.202348155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Vpc\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.204024925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.204790886Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.205856914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.206631943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.207366011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.20979764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.219071599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.220021626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.220806805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.221517998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.24088905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.247213251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.433623006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.434846887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.435564492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.442114831Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:47.599628136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:47.631727648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.669552211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":2,"elapsed":"3.337479559s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:47.71101548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"19d0d9ac-54de-4ef3-ba9f-6af140c8c324","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":3014592,"response_code":200}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:47.786740947Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"3b27283c-cabb-4c31-bab8-358606d519c3","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":8167217,"response_code":500}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.853963847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.855510319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.856425617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.857607734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.858626538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.866951031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:47.952439087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28054","req_id":"e2471c46-c6cc-48ec-8a60-f3f34466f645","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   be 9c 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:47.954366335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28054","req_id":"e2471c46-c6cc-48ec-8a60-f3f34466f645","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 48796,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 48796,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:47.965361893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28054","req_id":"e2471c46-c6cc-48ec-8a60-f3f34466f645","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.045916461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3b27283c-cabb-4c31-bab8-358606d519c3\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:06:47 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 157.650688ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:06:48.07270706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3b27283c-cabb-4c31-bab8-358606d519c3\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:06:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3b27283c-cabb-4c31-bab8-358606d519c3\" }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:48.774836797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:48.776478448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.78016501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":2,"elapsed":"4.489283016s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.781078444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.787599225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:48.789758202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62817","req_id":"f44854e5-5e90-4224-bc3e-9f6a20f764bd","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   61 0f 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   a............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.790875481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62817","req_id":"f44854e5-5e90-4224-bc3e-9f6a20f764bd","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 24847,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 24847,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:48.796028736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62817","req_id":"f44854e5-5e90-4224-bc3e-9f6a20f764bd","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:48.79655922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54145","req_id":"855f0c1b-e909-4b2d-b822-11e422da4107","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   fd e0 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.823207676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54145","req_id":"855f0c1b-e909-4b2d-b822-11e422da4107","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 64992,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 64992,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:48.828719202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54145","req_id":"855f0c1b-e909-4b2d-b822-11e422da4107","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:48.829247992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50586","req_id":"79947600-512c-4547-8649-6bf9200e10f1","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   83 89 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.830160595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50586","req_id":"79947600-512c-4547-8649-6bf9200e10f1","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33673,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33673,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:06:48.856441521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50586","req_id":"79947600-512c-4547-8649-6bf9200e10f1","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.971588512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":2,"elapsed":"4.472401989s"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:48.993848182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:48.994870352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:48.995988543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.025099961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.033953665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.034922459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.035669072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.03635274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.047951215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.166767043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"a2b7d067-becb-41f1-a1b8-cd4508f3fad6","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.201157646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"a2b7d067-becb-41f1-a1b8-cd4508f3fad6","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.22250816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.403960329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.405050123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.405883659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.411292902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.445017612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.445782721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.446503042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:49.447223703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.447902501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.454268956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:49.611521061Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.624730777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.704506588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:49.732981873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":3,"elapsed":"4.969921514s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:52.731504896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:52.732612847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:52.73339005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.734084349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.741215993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.774852328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"region_snapshot_replacement_step","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"step_garbage_collect_invoked_ok\":[],\"step_invoked_ok\":[],\"step_records_created_ok\":[],\"step_set_volume_deleted_ok\":[]}","iteration":1,"elapsed":"38.779187257s"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:52.844696013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:52.850981026Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.852975707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.900352521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:52.9519507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.955377966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:52.963915453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:57.170313225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.171180457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:57.175938828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:57.176790309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:57.177782419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.178486648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.201322372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.270394885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.454990458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:57.496780476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.599898126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.663048799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"no more instances in need of reincarnation","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:57.702773525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reincarnation_reason":"start saga failed","total_found":0}
{"msg":"instance reincarnation completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:57.76712858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","instances_changed_state":0,"instances_reincarnated":0,"instances_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.768650653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"changed_state\":[],\"disabled\":false,\"errors\":[],\"instances_found\":{\"Failed\":0,\"SagaUnwound\":0},\"instances_reincarnated\":[],\"restart_errors\":[]}","iteration":1,"elapsed":"43.789996386s"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:57.770193517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.771567937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":2,"elapsed":"13.2288973s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:57.773041092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.020106464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.021104126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.021935908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.022673603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.023650583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.056662427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:58.075328179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:43343"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.279293852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.386669318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.450690149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.472369676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.483766131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.502827338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.506818434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":5,"elapsed":"10.652601945s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.508778637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.510297995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.512616881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.514896167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.516373361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.529798585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.569886984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"b8804ab2-aa3c-4a50-9159-9a1fb6253b6c","remote_addr":"[::1]:43343","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.63546159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"b8804ab2-aa3c-4a50-9159-9a1fb6253b6c","remote_addr":"[::1]:43343","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:06:58.666745683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"b8804ab2-aa3c-4a50-9159-9a1fb6253b6c","remote_addr":"[::1]:43343","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":96919,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.91160809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8804ab2-aa3c-4a50-9159-9a1fb6253b6c\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:06:58 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:58.915138882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.923790144Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:58.956567593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:59.541917857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:59.54290639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:59.543672853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:59.54466323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:59.549954168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:59.596705811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:59.918610921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:06:59.950940921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:59.951895968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:06:59.959388789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.087101327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:00.087917427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:00.088643399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:00.089367918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.090048791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.095268648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:00.422998516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"a2b7d067-becb-41f1-a1b8-cd4508f3fad6","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":11256069,"response_code":500}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.526694826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.624717607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:00.625815999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.626549425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.632107895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"inventory_prune_one: nothing eligible for removal (too few)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.64179777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","candidates":"[]"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:00.644098349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.646093101Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.957939633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:00.973930372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.975713624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:00.99019758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.025984366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":3,"elapsed":"11.581098315s"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.149742339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a2b7d067-becb-41f1-a1b8-cd4508f3fad6\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:00 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 114.359964ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:01.176230337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a2b7d067-becb-41f1-a1b8-cd4508f3fad6\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a2b7d067-becb-41f1-a1b8-cd4508f3fad6\" }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.436153285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.438282684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.501085315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.535734973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.640123407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.641473247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.642211103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.650965129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.703065064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.704597738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.712205838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51434","req_id":"a510f5ad-b315-407c-a042-675c701f37e8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   d8 b5 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.728649737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51434","req_id":"a510f5ad-b315-407c-a042-675c701f37e8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55477,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55477,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:01.750328633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51434","req_id":"a510f5ad-b315-407c-a042-675c701f37e8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.781780348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52127","req_id":"4d4a8cff-eea2-44e9-ae3d-44326cbd7ff8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   f9 ef 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.784101248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52127","req_id":"4d4a8cff-eea2-44e9-ae3d-44326cbd7ff8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 63983,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 63983,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:01.789906137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52127","req_id":"4d4a8cff-eea2-44e9-ae3d-44326cbd7ff8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.79044677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62067","req_id":"602451cd-4f34-47ae-9274-9d7b6f530148","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   d2 a8 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.791360585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62067","req_id":"602451cd-4f34-47ae-9274-9d7b6f530148","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 53928,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 53928,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:01.796755099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62067","req_id":"602451cd-4f34-47ae-9274-9d7b6f530148","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.797357516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63420","req_id":"a20a1633-caa8-44cc-9ca2-bef4cd033c85","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   2d 32 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   -2..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.79826124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63420","req_id":"a20a1633-caa8-44cc-9ca2-bef4cd033c85","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 11570,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 11570,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:01.803182875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63420","req_id":"a20a1633-caa8-44cc-9ca2-bef4cd033c85","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.840351295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"183f8723-865a-4848-819d-83280a12d5f9","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.841586769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"183f8723-865a-4848-819d-83280a12d5f9","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"inventory collection failed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:01.881810456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"failed to collect inventory: looking up MGS addresses: proto error: no records found for Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.90819093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to collect inventory: looking up MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":2,"elapsed":"17.218609416s"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:01.956371701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.982013668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.982918716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":6,"elapsed":"3.473079812s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:01.984277143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:01.985056742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.0436034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:02.083148079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:02.084076183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:02.084953836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.085656571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.090921197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:02.526475895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.527534117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.539627816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.653087536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.801983356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:02.803061817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.803871175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.837241611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:02.874065196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.218836721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.220319416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.221116008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.246932521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.314438083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.395711489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.416827818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.460430797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.470619552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.613395968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.614322408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.615075915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.615885873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.620821807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.893313437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.894180519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.895040467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:03.89578696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.896485958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:03.967563664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.176163901Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.445650494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:04.446906058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.447804252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.452614435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:04.481597818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:04.482777942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.483768971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.554507784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.584733444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:04.585558463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:04.586281649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:04.58700714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.58768702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:04.601171902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:08.557703741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:08.563911279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:08.805382471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:08.856938355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:08.857860747Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:08.886144712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:08.904222686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.356695044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:09.357707635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.358805455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.365598967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.38344942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/artifacts-config","method":"PUT"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.387858857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.449471666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:09.452078172Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"183f8723-865a-4848-819d-83280a12d5f9","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":7611739,"response_code":500}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:09.48598272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:09.486765003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.362536194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:13.367541785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:13.368499177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.369332392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.376094933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:13.387339355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:56850"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:13.388568727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:36632"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:13.38916441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:63061"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:13.397055883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"974e1bea-98a9-44b0-af45-9abeef462ddb","remote_addr":"[::1]:43343","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.424069043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"974e1bea-98a9-44b0-af45-9abeef462ddb","remote_addr":"[::1]:43343","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.708956233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"183f8723-865a-4848-819d-83280a12d5f9\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:09 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 232.073407ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:13.710008553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"183f8723-865a-4848-819d-83280a12d5f9\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"183f8723-865a-4848-819d-83280a12d5f9\" }"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:13.747402811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"974e1bea-98a9-44b0-af45-9abeef462ddb","remote_addr":"[::1]:43343","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":350601,"response_code":204}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.750097302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.838228809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:13.839185609Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.839894326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.872206271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:13.928001324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"5d3c7671-b009-4570-b93d-b3ee798859d6","remote_addr":"[::1]:56850","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.928924518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"5d3c7671-b009-4570-b93d-b3ee798859d6","remote_addr":"[::1]:56850","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:13.946629511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/artifacts-config","method":"PUT","req_id":"d30f844f-b367-42ee-adb7-c7611d0016fc","remote_addr":"[::1]:36632","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:13.980577034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/artifacts-config","method":"PUT","req_id":"d30f844f-b367-42ee-adb7-c7611d0016fc","remote_addr":"[::1]:36632","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.015356639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"814428ef-7586-428e-bd0e-596547cd278e","remote_addr":"[::1]:63061","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.016425763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"814428ef-7586-428e-bd0e-596547cd278e","remote_addr":"[::1]:63061","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.020241521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"974e1bea-98a9-44b0-af45-9abeef462ddb\", \"date\": \"Wed, 04 Mar 2026 07:07:13 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.021523597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":1,"elapsed":"60.040048419s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.022146094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.022735775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.023263252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.0239521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.026961077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.027711417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.068467041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.108381454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"5d3c7671-b009-4570-b93d-b3ee798859d6","remote_addr":"[::1]:56850","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":180748,"response_code":204}
{"msg":"No ledger in /var/tmp/omicron_tmp/.tmpjVbLmb/artifact-config.json","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.112952563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"Failed to read ledger: Not found in storage","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.114531268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","path":"/var/tmp/omicron_tmp/.tmpjVbLmb/artifact-config.json"}
{"msg":"No ledger in /var/tmp/omicron_tmp/.tmpZywEx5/artifact-config.json","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.115943052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"Failed to read ledger: Not found in storage","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.117126543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","path":"/var/tmp/omicron_tmp/.tmpZywEx5/artifact-config.json"}
{"msg":"Writing ledger to /var/tmp/omicron_tmp/.tmpjVbLmb/.artifact-config.json.tmp","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.118721811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.122301711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"814428ef-7586-428e-bd0e-596547cd278e","remote_addr":"[::1]:63061","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":107291,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.162125915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.168330538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.170038019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.171787554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.185339399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.212273252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.213492383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.215437997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:14.216524826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.217149096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":2,"elapsed":"440.776µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.217819297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.21843012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.21893867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.219538832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.220117872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.22082691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.23407383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.239187736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.23997495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.241037159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:14.241740226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.251035547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":2,"elapsed":"9.851586ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.251953991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.25257285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 06:57:14.251182058 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.253476435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.254175183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.259252967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.264060105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.264849272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.265545014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.266794055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.26747667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.273525614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.279036189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.279836268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.280517481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":2,"elapsed":"534.542µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.281242621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.281820318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.282503094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.305307576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.312553517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.313282735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.31398012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.321118628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.363842862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.371436774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.409747883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.417657422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.421445064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.423506659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.425762008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.454470866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.473783783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.474677398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.475478329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.476571801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.477308674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.477835751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.47981394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.481575859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.537674095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.544578498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.545449778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.546385206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.547239573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.547802651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.548498794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63521","req_id":"d3b6b6de-89e2-45eb-b646-314103628382","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   83 d8 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.563455569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63521","req_id":"d3b6b6de-89e2-45eb-b646-314103628382","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33752,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33752,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:14.569722606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63521","req_id":"d3b6b6de-89e2-45eb-b646-314103628382","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.570539789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58392","req_id":"5a52d686-ea9b-4e93-b60d-cf00724a6920","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   f8 8d 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.571624233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58392","req_id":"5a52d686-ea9b-4e93-b60d-cf00724a6920","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 63629,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 63629,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:14.577853374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58392","req_id":"5a52d686-ea9b-4e93-b60d-cf00724a6920","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.578698723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59587","req_id":"00206693-054f-459a-9f7a-73ff5755aa78","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   08 c8 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.638723698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59587","req_id":"00206693-054f-459a-9f7a-73ff5755aa78","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 2248,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 2248,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:14.661412319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59587","req_id":"00206693-054f-459a-9f7a-73ff5755aa78","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.662215514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"5d3c7671-b009-4570-b93d-b3ee798859d6\", \"date\": \"Wed, 04 Mar 2026 07:07:14 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.663311481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.663954498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.66468554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":1,"elapsed":"60.537278409s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.665653553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.669243553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.669918072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.670623994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.671337832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.672008143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.672567945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.799190177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"814428ef-7586-428e-bd0e-596547cd278e\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:07:14 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.800300542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.80091411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:14.801616816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.802198331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":1,"elapsed":"60.828233859s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.802793353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.803357473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.804026282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.841421572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.918421269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.919472046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":7,"elapsed":"10.985209562s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.922344561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.923040794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.923871113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.924607015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.92634674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.927398218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.960590311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c8acf070-3c71-4f94-95fc-c3ee13b620b4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.961640877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c8acf070-3c71-4f94-95fc-c3ee13b620b4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.962339775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"179255e4-c221-453e-b61b-a3f9b5ad4611","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.963411704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"179255e4-c221-453e-b61b-a3f9b5ad4611","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.988039567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:14.989216314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:14.989970642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.011400885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.063411282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.064210399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.065097681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.080564589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.119567733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.129779064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.130533342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":3,"elapsed":"210.558µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.131190517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:15.131990315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.13254173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":3,"elapsed":"135.599µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.133182343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.134045247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.134745227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.135462922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.13614156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.159561524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.160772249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.16138221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.162696079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.163445187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.164119476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.203178391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.209519204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:15.210293231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.243228634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":3,"elapsed":"33.890981ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.32262903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.323493677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.324795763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":4,"elapsed":"15.211192982s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.325506825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c8acf070-3c71-4f94-95fc-c3ee13b620b4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.327157614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c8acf070-3c71-4f94-95fc-c3ee13b620b4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.360927363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c8acf070-3c71-4f94-95fc-c3ee13b620b4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:15.362996012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.408906183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.409691102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.410942447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.440533518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.441379578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.441961915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.472641346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:15312","req_id":"bab2f02e-fe3b-4b6f-a166-0622e0b952e8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   aa f6 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   ............._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.473835318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:15312","req_id":"bab2f02e-fe3b-4b6f-a166-0622e0b952e8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43766,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43766,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.479875504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:15312","req_id":"bab2f02e-fe3b-4b6f-a166-0622e0b952e8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.480426258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10777","req_id":"9e6886f9-d030-4015-bf76-70b71117e59e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   e8 0f 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   ............._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.481326576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10777","req_id":"9e6886f9-d030-4015-bf76-70b71117e59e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 59407,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 59407,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.552408931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10777","req_id":"9e6886f9-d030-4015-bf76-70b71117e59e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.561302204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:29879","req_id":"09c865e9-d0f6-4ae2-8791-0aecca26cb91","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   d9 2a 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .*..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.562741843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:29879","req_id":"09c865e9-d0f6-4ae2-8791-0aecca26cb91","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55594,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55594,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.568898872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:29879","req_id":"09c865e9-d0f6-4ae2-8791-0aecca26cb91","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.569447241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26464","req_id":"2fcf2278-21a1-4692-b083-69ce50514020","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   17 7a 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .z..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.570351868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26464","req_id":"2fcf2278-21a1-4692-b083-69ce50514020","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6010,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6010,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.576165945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26464","req_id":"2fcf2278-21a1-4692-b083-69ce50514020","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:15.576855945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.577854168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":2,"elapsed":"1.351040508s"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:15.578699657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.579407833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":2,"elapsed":"1.330852041s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.580246649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.580927341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.594735085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.595773748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":2,"elapsed":"1.107374747s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.596700117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.597447221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.60004976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.600786123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.601363139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:15.633428503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:25231","req_id":"25148571-d6a8-4ee6-95ba-4229b6d10077","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   db 26 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .&..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.635361051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:25231","req_id":"25148571-d6a8-4ee6-95ba-4229b6d10077","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56102,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56102,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.646471137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:25231","req_id":"25148571-d6a8-4ee6-95ba-4229b6d10077","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:15.796928056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.799345798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":3,"elapsed":"509.383558ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:15.958485653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.577628331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Vpc\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.579024875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.579623905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.580327402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.58649927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.617914433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.618840352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.619611103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.620731398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.628635275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.656996804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.657914697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.65868676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.659397502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.668067421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Writing ledger to /var/tmp/omicron_tmp/.tmpZywEx5/.artifact-config.json.tmp","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:16.681852439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.689483365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.792227913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.793185184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.793901217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:16.794620685Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.795296347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.802373042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.817679582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":4,"elapsed":"12.232607986s"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:16.902246118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 06:57:14.251182058 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.903824453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T06:57:14.251182058Z\",\"pruned\":[]}","iteration":2,"elapsed":"2.652077724s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:16.952240271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.285698847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:17.286690096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.287523952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.295545965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.306276366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":2,"elapsed":"2.993493424s"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.312171164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:17.312878078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.313540012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":2,"elapsed":"3.033323951s"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:17.317383626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.318052164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":2,"elapsed":"2.896580166s"}
{"msg":"Starting delete reconciler","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:17.335202305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"ArtifactStore","server":"b6d65341-167c-41df-9b5c-41cded99c229","component":"SledAgent","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","generation":1}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:17.341201851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/artifacts-config","method":"PUT","req_id":"d30f844f-b367-42ee-adb7-c7611d0016fc","remote_addr":"[::1]:36632","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":3395230,"response_code":204}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.475176055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.583154409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:17.584241017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.584988301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.590305962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.704701992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/artifacts-config\", status: 204, headers: {\"x-request-id\": \"d30f844f-b367-42ee-adb7-c7611d0016fc\", \"date\": \"Wed, 04 Mar 2026 07:07:17 GMT\"} })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.705719974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/artifacts","method":"GET"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:17.742924335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.164015166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.199612635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.200510949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.201255889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.201944076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.211123948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.391628635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/artifacts","method":"GET","req_id":"a8cbcd2c-d53c-4d1c-a5f7-b7df776f2aaf","remote_addr":"[::1]:36632","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.39256269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/artifacts","method":"GET","req_id":"a8cbcd2c-d53c-4d1c-a5f7-b7df776f2aaf","remote_addr":"[::1]:36632","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.393818634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:18.435275261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/artifacts","method":"GET","req_id":"a8cbcd2c-d53c-4d1c-a5f7-b7df776f2aaf","remote_addr":"[::1]:36632","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":43817,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.442659827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.651825107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.652831896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.653557437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.659422335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.703444848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":3,"elapsed":"3.58362781s"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:18.858707722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.86466792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:18.998392999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.003855758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.01226996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.031682374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.220802623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.432903823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.495529052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.515402172Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.639569448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.69588695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/artifacts\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8cbcd2c-d53c-4d1c-a5f7-b7df776f2aaf\", \"content-length\": \"26\", \"date\": \"Wed, 04 Mar 2026 07:07:18 GMT\"} })"}
{"msg":"Successfully got artifact list","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:19.698923771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled":"http://[::1]:33621"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.717086764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_artifact_replication","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"generation\":1,\"last_run_counters\":{\"copy_err\":0,\"copy_ok\":0,\"list_err\":0,\"list_ok\":1,\"put_config_err\":0,\"put_config_ok\":1,\"put_err\":0,\"put_ok\":0},\"lifetime_counters\":{\"copy_err\":0,\"copy_ok\":0,\"list_err\":0,\"list_ok\":1,\"put_config_err\":0,\"put_config_ok\":1,\"put_err\":0,\"put_ok\":0},\"local_repos\":0,\"request_debug_ringbuf\":[{\"generation\":1,\"operation\":\"put_config\",\"target_sled\":\"b6d65341-167c-41df-9b5c-41cded99c229\",\"time\":\"2026-03-04T07:07:17.704330424Z\"},{\"operation\":\"list\",\"target_sled\":\"b6d65341-167c-41df-9b5c-41cded99c229\",\"time\":\"2026-03-04T07:07:19.698136487Z\"}]}","iteration":1,"elapsed":"65.718206918s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:19.783512801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c8acf070-3c71-4f94-95fc-c3ee13b620b4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":4829085,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.83699438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.870791133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.87183197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.872523814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.873313853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.910842939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.91929248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58408","req_id":"d84b2977-af60-49f8-a01c-3b9702b8a9a5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   ed 7f 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.920471823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58408","req_id":"d84b2977-af60-49f8-a01c-3b9702b8a9a5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 60799,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 60799,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:19.926334707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58408","req_id":"d84b2977-af60-49f8-a01c-3b9702b8a9a5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:19.927017383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:2464","req_id":"1ce1bc86-51e9-49b8-a974-e29f58a18813","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   28 c4 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   (............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:19.927991908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:2464","req_id":"1ce1bc86-51e9-49b8-a974-e29f58a18813","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 10436,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 10436,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:19.954201062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:2464","req_id":"1ce1bc86-51e9-49b8-a974-e29f58a18813","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:20.073006038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:20.076472092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:20.134975274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.136626494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":2,"elapsed":"34.971148802s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.139052101Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:20.141851511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.147475811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.182237711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:20.259503478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:20.289348813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.290899733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":3,"elapsed":"5.304716423s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:20.347778379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:20.348719027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.34975254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":3,"elapsed":"5.285343757s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.413052739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.737338217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.740504145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.92587195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:20.926912006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.927624561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:20.93679272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:21.018923709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.019970337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.091332017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.142770308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.522412289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.538411214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.647209175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:21.68858455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.689754945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.702193669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:21.83782803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:21.83872948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:21.839763634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.840585847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:21.886593161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:22.251721507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:22.539159151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:22.544021789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:22.597890086Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:22.728880093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.210079775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.210909553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.211913858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.29328608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.302743242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.303661265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.304404722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.305377815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.311139869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.373971427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.45953813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":8,"elapsed":"8.580638994s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.460366966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.460976557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.461692659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:23.462727965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.463428416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.468563285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.786025009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":5,"elapsed":"3.948943657s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:23.971319529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":3,"elapsed":"8.845449991s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:24.332409374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:24.862348366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:24.863329354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:24.864046779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:24.910135084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:25.039683498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:25.040649737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:25.175351306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:25.214274861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:25.51565644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:25.516663249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:25.517492256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:25.569612341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.193666112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.62019825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:26.718898537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.761221625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.768960825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:26.794693241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.795669631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":2,"elapsed":"12.431955304s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.796423247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:26.907165641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.019640165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53784","req_id":"bb1b8e3a-9ef9-4222-944f-1f725a1e7aaa","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   ca 99 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.020888634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53784","req_id":"bb1b8e3a-9ef9-4222-944f-1f725a1e7aaa","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 51865,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 51865,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:27.08413679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53784","req_id":"bb1b8e3a-9ef9-4222-944f-1f725a1e7aaa","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.084830588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53412","req_id":"3014ade4-781f-46b7-bce9-902bd6267963","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   99 fa 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.085879211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53412","req_id":"3014ade4-781f-46b7-bce9-902bd6267963","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 39418,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 39418,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:27.120134785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53412","req_id":"3014ade4-781f-46b7-bce9-902bd6267963","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.123023833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63070","req_id":"a2b8614d-4181-448d-b378-7af6fd51af68","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   1c 5e 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .^..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.124464094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63070","req_id":"a2b8614d-4181-448d-b378-7af6fd51af68","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 7262,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 7262,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:27.172376742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63070","req_id":"a2b8614d-4181-448d-b378-7af6fd51af68","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.183000151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":5,"elapsed":"11.776500748s"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:27.372468741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.390883304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":2,"elapsed":"13.228621469s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.406422055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.420380458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.467711592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.61334406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.632609152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":9,"elapsed":"4.172589285s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.636585519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.638770719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.651322047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:27.949463861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:27.993807421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":2,"elapsed":"13.775449404s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.996567924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.99849946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:27.999979459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.00756761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.045614635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:28.215072051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.232964557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":2,"elapsed":"13.762098589s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:28.516183794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:28.538544183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:28.539479271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.540507142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.574207353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.696089121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:28.696906173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:28.697643618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:28.698376654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.69906461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.704407642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:28.929735762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.176223347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.304817285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Err(reqwest::Error { kind: Request, url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", source: TimedOut })"}
{"msg":"failed to contact nexus, will retry in 244.903105ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:29.30612374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Communication Error: error sending request for url (http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229)"}
{"msg":"request handling cancelled (client disconnected)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:29.316750345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"179255e4-c221-453e-b61b-a3f9b5ad4611","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":14355526}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:29.550578157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:29.551339679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.551660997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.589978419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.680744949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.937565565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:29.938899945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.939630597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:29.94472756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.101107142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.102609797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:30.213071112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:30.236585053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.238433384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.252425944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:30.308479441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:47529"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:30.498068641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"5e0133a5-e77a-4b19-a7a7-65c76e2fe03a","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.49903977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"5e0133a5-e77a-4b19-a7a7-65c76e2fe03a","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.499743096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:30.500353128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:30.501042758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:30.501757627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.502432788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.524003149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:30.632662535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.67239146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:30.673452858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":10,"elapsed":"1.976653517s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:31.02794567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:31.731681327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"VpcSubnet\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:31.766795908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:31.810104371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:31.813115251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:31.854867024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:31.87852457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:31.941759059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:31.950221015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.385113167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:32.386096299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.387103029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.419050719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:32.479391661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.514020637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":3,"elapsed":"17.354454635s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:32.515479916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"5e0133a5-e77a-4b19-a7a7-65c76e2fe03a","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":2016996,"response_code":500}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:32.708445471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:32.7094013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.710128213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.7173416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:32.80386897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:32.804723607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:32.806631166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.807784547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":3,"elapsed":"12.665744191s"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:32.921090011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e0133a5-e77a-4b19-a7a7-65c76e2fe03a\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:32 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 170.762982ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:32.923067839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e0133a5-e77a-4b19-a7a7-65c76e2fe03a\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5e0133a5-e77a-4b19-a7a7-65c76e2fe03a\" }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.18146135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":6,"elapsed":"2.682257915s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.274854256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.275987957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.373205209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.374593306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.375569184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.380414918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.619570761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"b69c7530-8026-4c0d-af2c-fa618bc98378","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.620685315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"b69c7530-8026-4c0d-af2c-fa618bc98378","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:33.651667879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.727558915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.728726975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.729651972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.730666076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.732330501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.760989571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:33.933938611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.935706923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:33.941863821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:34.202327494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:34.203197782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":2,"elapsed":"19.93875635s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:34.24984625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:34.821122421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:34.928525221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:35.007206449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:35.146344919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:39.502702375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.504268335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.508620007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.510202951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":2,"elapsed":"25.004971009s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.520763926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.701796131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:39.702774464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.703614613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.710301975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed after handler was already cancelled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:39.783888903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"179255e4-c221-453e-b61b-a3f9b5ad4611","remote_addr":"127.0.0.1:60153","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","response_code":500}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:39.916689937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:39.917665986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:39.918421346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.919424939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:39.927821696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.017342021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.018340764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.019084712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.019764512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.03403805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:40.041566332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:50638"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.072874012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.073718018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.074562195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.075298226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.076098756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.080826216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.107303692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.108086587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.108797399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.175798909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:40.263600621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"b69c7530-8026-4c0d-af2c-fa618bc98378","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":6643974,"response_code":500}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.269824592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.27108233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.296353571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.348564366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.364521648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":11,"elapsed":"6.630131527s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.368982057Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.370148574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.371408887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.374942996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.376603614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.422129662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:40.557519949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"6342f8d2-767c-4dc0-8b85-d2bdf2bfed2c","remote_addr":"[::1]:50638","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.586013339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"6342f8d2-767c-4dc0-8b85-d2bdf2bfed2c","remote_addr":"[::1]:50638","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.656601804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b69c7530-8026-4c0d-af2c-fa618bc98378\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:40 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 154.462145ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:40.679070749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b69c7530-8026-4c0d-af2c-fa618bc98378\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b69c7530-8026-4c0d-af2c-fa618bc98378\" }"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:40.718397244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"6342f8d2-767c-4dc0-8b85-d2bdf2bfed2c","remote_addr":"[::1]:50638","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":160943,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:40.903607647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.162637752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.180850454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.190875104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.192307489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.193437764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.251979733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.285784502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.287430642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.288617589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.289853974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.307391095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.371655018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.630389416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.631428971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.6324107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.639316967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.699313866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.751748565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.754224142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64058","req_id":"7fe72f6f-5303-48c8-8ed3-bc02562a91a3","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   6f 20 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   o ..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.755810723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64058","req_id":"7fe72f6f-5303-48c8-8ed3-bc02562a91a3","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 28448,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 28448,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:41.762070966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64058","req_id":"7fe72f6f-5303-48c8-8ed3-bc02562a91a3","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.762633394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62053","req_id":"6fbedcd8-edb3-4352-83a4-bf9f70bfe311","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   b6 05 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.763547038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62053","req_id":"6fbedcd8-edb3-4352-83a4-bf9f70bfe311","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 46597,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 46597,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:41.768792486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62053","req_id":"6fbedcd8-edb3-4352-83a4-bf9f70bfe311","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.769294343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50000","req_id":"3e61bd9b-8eab-4346-9c68-f1e3873e5900","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   7c 01 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   |............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.770197076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50000","req_id":"3e61bd9b-8eab-4346-9c68-f1e3873e5900","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 31745,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 31745,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:41.776915469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50000","req_id":"3e61bd9b-8eab-4346-9c68-f1e3873e5900","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.799136701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6342f8d2-767c-4dc0-8b85-d2bdf2bfed2c\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:07:40 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:41.800171978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.800879112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.808272877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:41.853810819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.430925075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:42.431941995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.432685812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.438708634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:42.675701021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.694850403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.701331194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.781944558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:42.91329429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.08827175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.124315075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":12,"elapsed":"2.75496158s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:43.154695352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f0f352a3-e59c-4f60-a7f9-be90112891a5","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.155763554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f0f352a3-e59c-4f60-a7f9-be90112891a5","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:43.257498418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.318144085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":6,"elapsed":"3.245331836s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:43.34637288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.377219824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.382969544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:43.417574422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.420478239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.467235493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:43.730478766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:43.788806349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:43.829700217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.895830056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:43.972447134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:44.55038683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:44.551768996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:44.574866058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:44.660646044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:44.775605052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:45.254818059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:45.256556141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:45.257553472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.258278902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.271409982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.401323471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.650642218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:45.651988853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.652904261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.701744652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.777532633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.92972546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:45.931434475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.9333455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:45.938750946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:46.159736141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:46.161973013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:46.163590806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:46.198495861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.013625021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:50.019185605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:55178"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.026197882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"8938e465-2813-4751-92d8-a16ae60e9bb1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.08064582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"8938e465-2813-4751-92d8-a16ae60e9bb1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.083728042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.084461859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.085171479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.099962735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.106104333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.111705658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.112609322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.119573223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.124720817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.13053805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.131557816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":4,"elapsed":"363.412µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.132243267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:50.132963517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.133514662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":4,"elapsed":"135.99µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.13416796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.1347573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.135456789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.136187511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.136881569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.156990287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.182350775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.18324961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.184024679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.184763837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.185465801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.201045223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.22073052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:50.22850975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.231567874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":4,"elapsed":"9.770034ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.233757753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.235586635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.255571798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.256353871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.257091837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.257830043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.258534642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.285950081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.319978084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.344397333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.345640842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.346786498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.347860891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.354263187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.573647492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"8938e465-2813-4751-92d8-a16ae60e9bb1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.575186559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"8938e465-2813-4751-92d8-a16ae60e9bb1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.677356848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"8938e465-2813-4751-92d8-a16ae60e9bb1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:50.757361463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"e1937f4d-337a-450a-aebe-047988e2901b","remote_addr":"[::1]:55178","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:50.758283294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"e1937f4d-337a-450a-aebe-047988e2901b","remote_addr":"[::1]:55178","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.44847578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.452048246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"e1937f4d-337a-450a-aebe-047988e2901b","remote_addr":"[::1]:55178","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":3693878,"response_code":200}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.460264624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f0f352a3-e59c-4f60-a7f9-be90112891a5","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":11307295,"response_code":500}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.476971004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.47801129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.521817393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:40640"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.530811756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"55bc3862-589b-4ed7-9112-5914fafc1ce1","remote_addr":"[::1]:50638","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.538081054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"55bc3862-589b-4ed7-9112-5914fafc1ce1","remote_addr":"[::1]:50638","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.548734933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.550558134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.551879138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.553413786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.566920892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.634923531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.643896462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7176","req_id":"9503844a-c196-455b-8597-0ecea2cc81c6","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   a8 8c 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.672107301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7176","req_id":"9503844a-c196-455b-8597-0ecea2cc81c6","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43148,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43148,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:54.680894433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7176","req_id":"9503844a-c196-455b-8597-0ecea2cc81c6","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.681493753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0f352a3-e59c-4f60-a7f9-be90112891a5\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:54 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 291.642135ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:54.682347709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0f352a3-e59c-4f60-a7f9-be90112891a5\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:07:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0f352a3-e59c-4f60-a7f9-be90112891a5\" }"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.688116947Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.689015973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.689742446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.698851277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.736584919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1937f4d-337a-450a-aebe-047988e2901b\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:07:54 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.739000396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.742938427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.745473873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.750636687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":2,"elapsed":"39.950090136s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.751682213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.752595157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.75349834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.758635393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.771556135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"55bc3862-589b-4ed7-9112-5914fafc1ce1","remote_addr":"[::1]:50638","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":240480,"response_code":204}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:54.793674453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.796685813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":4,"elapsed":"4.56219249s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.80073213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"cb78b638-43f0-4abc-b7d4-8f8b3d464568","remote_addr":"[::1]:40640","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.802723835Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"cb78b638-43f0-4abc-b7d4-8f8b3d464568","remote_addr":"[::1]:40640","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.808921945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"55bc3862-589b-4ed7-9112-5914fafc1ce1\", \"date\": \"Wed, 04 Mar 2026 07:07:54 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.880490016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":2,"elapsed":"40.858900277s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.911043557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.950080227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.950904605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.951643212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:54.952372831Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.953059915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:54.960780138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:54.979798891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"cb78b638-43f0-4abc-b7d4-8f8b3d464568","remote_addr":"[::1]:40640","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":178956,"response_code":204}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.041848527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.064660403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":4,"elapsed":"4.939841822s"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.067374924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.083251356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.084783529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.09050366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57838","req_id":"5819a5f9-ca4b-4b57-8e0f-a228bccec3a9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   26 95 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   &............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.091489358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57838","req_id":"5819a5f9-ca4b-4b57-8e0f-a228bccec3a9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 9877,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 9877,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:55.103496455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57838","req_id":"5819a5f9-ca4b-4b57-8e0f-a228bccec3a9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.10410312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61147","req_id":"5efc1b11-3be5-4a54-879a-0b5e86fe1753","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   f5 45 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .E..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.105036213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61147","req_id":"5efc1b11-3be5-4a54-879a-0b5e86fe1753","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62789,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62789,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:55.122701267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61147","req_id":"5efc1b11-3be5-4a54-879a-0b5e86fe1753","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.123969155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61701","req_id":"6d4e623f-b915-495a-a89d-e1a122c9d811","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   c0 39 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .9..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.125769872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61701","req_id":"6d4e623f-b915-495a-a89d-e1a122c9d811","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 49209,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 49209,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:55.132676499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61701","req_id":"6d4e623f-b915-495a-a89d-e1a122c9d811","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.217901753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21416","req_id":"fa4db299-8759-48df-989b-47c9bfcb9d8a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   c0 b7 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.220173786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21416","req_id":"fa4db299-8759-48df-989b-47c9bfcb9d8a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 49335,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 49335,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:55.235223105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21416","req_id":"fa4db299-8759-48df-989b-47c9bfcb9d8a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.243671895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"cb78b638-43f0-4abc-b7d4-8f8b3d464568\", \"date\": \"Wed, 04 Mar 2026 07:07:54 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.245641487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:55.246363511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.248386729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":2,"elapsed":"40.666974286s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.250996622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.472402954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.498929688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.502569537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.515150784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.531118498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.73133791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:55.732538224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.73326651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.802652426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:55.826712902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.101311137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.109858753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.112165054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.120401081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.27112409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"7637f03d-7a54-42de-b2fd-fdd4266c5008","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.277635702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"7637f03d-7a54-42de-b2fd-fdd4266c5008","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:56.296612101Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:56.297890941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.298922981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":4,"elapsed":"6.213713827s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.301158621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.301870374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.302590695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.303327288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.304558724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.421392014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.428379933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:23008","req_id":"6fa5dfeb-23ce-41c3-8246-dda779d8f7b4","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   06 35 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .5..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.42984894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:23008","req_id":"6fa5dfeb-23ce-41c3-8246-dda779d8f7b4","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 1589,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 1589,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:56.436029345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:23008","req_id":"6fa5dfeb-23ce-41c3-8246-dda779d8f7b4","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:56.62438742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.683852783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.920692536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:56.939970764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:56.941143985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:57.005773885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:57.139409356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:07:57.140537126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:57.141254491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":4,"elapsed":"7.033875677s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:57.18093025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:57.706449465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"VpcSubnet\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:57.707869645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:57.708450539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:57.709561736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:57.710305703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:57.711003569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:57.723443386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:57.827204253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"8938e465-2813-4751-92d8-a16ae60e9bb1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":7800851,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.091005824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.444484132Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:58.445604106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.471277566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.480105178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:58.905590135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:58.950717121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:07:58.951518984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.952084838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":4,"elapsed":"18.844035505s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.952800239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:58.953682591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.954399284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.960211528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:58.971879562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.060517143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.081352424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.082741132Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.08396864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.085228602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.093521934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.173653121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":4,"elapsed":"9.041978664s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.177740278Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.178705375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":13,"elapsed":"8.923398824s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.179386418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.179985658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.18070163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.181440647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.182141559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.186627659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.287225605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.288713099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.289965888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.291016244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.349541319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read target blueprint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:07:59.401048427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: no target blueprint set"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.402039916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read target blueprint: Internal Error: no target blueprint set\"}","iteration":2,"elapsed":"4.852281299s"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.403125241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.404268522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.405388727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.437737965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.478437498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":7,"elapsed":"9.158162645s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.596238871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.874482255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:07:59.875574033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.876600953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:07:59.910887759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:00.39521827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:00.396304828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:00.397047412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:00.40676365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:05.824956235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:05.826128814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:05.832490879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.578974527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:06.581124087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.582482034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.596200931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:06.677232666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.701102123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":4,"elapsed":"16.518118431s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.702197639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.824312949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:06.842299472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"7637f03d-7a54-42de-b2fd-fdd4266c5008","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":10571283,"response_code":500}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:06.932855324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.087453689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:07.08850662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.089234706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.094571215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.132799289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.59151201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:07.612647006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.61437589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.627823297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:07.822582525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.849415568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.858543266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:07.871345713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:37034"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:07.90420782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:07.906633337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:07.926562227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:07.984927656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.066816684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.132464396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.180570499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7637f03d-7a54-42de-b2fd-fdd4266c5008\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:06 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 610.451613ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:08.218835486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7637f03d-7a54-42de-b2fd-fdd4266c5008\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7637f03d-7a54-42de-b2fd-fdd4266c5008\" }"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.270852025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.272056257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":14,"elapsed":"9.092627997s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.272771668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:08.27337669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:08.274114585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:08.275134541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.275883859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.286674961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:08.414938854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"11dac7cd-bfd6-44c9-a007-c495323d4ef1","remote_addr":"[::1]:37034","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.422175567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"11dac7cd-bfd6-44c9-a007-c495323d4ef1","remote_addr":"[::1]:37034","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:08.800848371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"11dac7cd-bfd6-44c9-a007-c495323d4ef1","remote_addr":"[::1]:37034","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":386124,"response_code":200}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:08.966979482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":7,"elapsed":"12.665274205s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.146747255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.148002679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.148867146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.149563368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.154545884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"resolved names for firewall rules","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.215575013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","subnet_networks":"{Name(\"nexus\"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })], Name(\"external-dns\"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })]}","subnet_interfaces":"{}","vpc_interfaces":"{}","instance_interfaces":"{}"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.216729155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.217788008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.218552557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.219268378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.22466701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.233844127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.569827748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.571427345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.572542009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.591264253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.652041892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.653377866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.655996687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57911","req_id":"ad280073-bc90-4995-aade-639960805778","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   da a2 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.657236299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57911","req_id":"ad280073-bc90-4995-aade-639960805778","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55970,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55970,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:09.663362636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57911","req_id":"ad280073-bc90-4995-aade-639960805778","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.663950824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56984","req_id":"127fb8ef-14aa-4afa-86ea-4d76f4cee4b5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   0a 32 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .2..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.683978331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56984","req_id":"127fb8ef-14aa-4afa-86ea-4d76f4cee4b5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 2610,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 2610,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:09.776638511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56984","req_id":"127fb8ef-14aa-4afa-86ea-4d76f4cee4b5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.777273352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58842","req_id":"3a8c691a-22b8-4788-b689-2cb256e9a6b7","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   bb d4 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.779846773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58842","req_id":"3a8c691a-22b8-4788-b689-2cb256e9a6b7","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 48084,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 48084,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:09.826079505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58842","req_id":"3a8c691a-22b8-4788-b689-2cb256e9a6b7","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.834557723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11dac7cd-bfd6-44c9-a007-c495323d4ef1\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:08:08 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.835608971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.836336416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.860996844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:09.980772568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:09.981674139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.01389374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.023829433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.162177764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.169964699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.459552365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:10.460717929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.461464622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.483989559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.533075367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.548889145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":15,"elapsed":"2.277591785s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:10.583805101Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"fcff3e1b-37c2-4272-b749-4bbdf8e95d9f","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:10.58469112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"fcff3e1b-37c2-4272-b749-4bbdf8e95d9f","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:10.794282135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"fetched allowlist from DB","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.023853292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","allowed":"AllowList { id: 001de000-a110-4000-8000-000000000000, time_created: 2026-03-04T05:46:29.930974Z, time_modified: 2026-03-04T05:46:29.930974Z, allowed_ips: None }"}
{"msg":"Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified.","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.025571936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"resolved firewall rules for sled agents","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.026813321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_agent_rules":"[]"}
{"msg":"resolved 0 rules for sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.027473843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.129359116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.130315796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.131073791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.13176287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.13698425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.459745781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.460902979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.461666115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.462406145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.46309356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.483967628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.676214235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.909552053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.910531207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.911259895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:11.91203301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.91348304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:11.942201507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.148661083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.399049375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:12.400182215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.40092488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.466989459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.744184271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.874249291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:12.875218586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.87593576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.91512326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:12.923969118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":8,"elapsed":"1.465508269s"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:13.211252227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:13.212176572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:13.212892023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.213547034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":5,"elapsed":"14.260338695s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.214245321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.627014304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:13.628324567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.629051591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.635342565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.747321404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:13.748147896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:13.748904418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:13.749651632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.750929941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:13.757099224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.023451371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":8,"elapsed":"6.11850805s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.067194418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.294095643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.295030088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.29614858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.311700307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.359170336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.35992735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.360633312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.393270552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.418050517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.427762336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.428711791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.429614063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.451765869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.457386672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:14.458354725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.459794614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":3,"elapsed":"433.541µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.460891924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.461773244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.462563784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.463666665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.464537675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.465838077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.516751295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.526339648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.570894137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.572675756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:14.573907712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.575547078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":3,"elapsed":"423.551µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.576637524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:14.577507151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 06:58:14.573112433 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.578544281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.579423026Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.681957488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.69184197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.737017963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.852251105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:14.936071199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.952565077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:14.977922319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.088165671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:15.101089008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.149291933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":3,"elapsed":"61.720332ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.190714852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.235285403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.264540726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.283938161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.290339475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.291351805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.292427752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.299766488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.594118325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"a12812d8-39dc-4172-94e7-081ddb110aa8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.595536832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"a12812d8-39dc-4172-94e7-081ddb110aa8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.596878026Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.598129481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.598865122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.618171967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.625160958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.626058922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.626785575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.730858803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.736966653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.737807403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.738395251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.739007216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.739554653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.740386746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.741255261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.742004429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.742768176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.743659797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.785284927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.803515715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:15.804331575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.804912499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.807610237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:15.859731634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.860473037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.861215702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.861950691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.862649139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.869260409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:15.883363359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"fcff3e1b-37c2-4272-b749-4bbdf8e95d9f","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":5299702,"response_code":500}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:15.947559908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"a12812d8-39dc-4172-94e7-081ddb110aa8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.968745044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"a12812d8-39dc-4172-94e7-081ddb110aa8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:15.984938435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"a12812d8-39dc-4172-94e7-081ddb110aa8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.038197182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.039141788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.039941015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.060227415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.091843227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.115273882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.116155433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.116919871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.117933164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.123609798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.148256869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61040","req_id":"9285d150-e531-4002-bea0-3d8c6c1edc07","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   88 0c 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   ............._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.14935504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61040","req_id":"9285d150-e531-4002-bea0-3d8c6c1edc07","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 34828,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 34828,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:16.156784826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61040","req_id":"9285d150-e531-4002-bea0-3d8c6c1edc07","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.157379287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64831","req_id":"25d95abe-a2b4-43f2-a3e7-c958faaa30c8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   ee ec 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   ............._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.158294344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64831","req_id":"25d95abe-a2b4-43f2-a3e7-c958faaa30c8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 61164,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 61164,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:16.17060298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64831","req_id":"25d95abe-a2b4-43f2-a3e7-c958faaa30c8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.171484762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fcff3e1b-37c2-4272-b749-4bbdf8e95d9f\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:15 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 319.962341ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:16.172537112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fcff3e1b-37c2-4272-b749-4bbdf8e95d9f\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fcff3e1b-37c2-4272-b749-4bbdf8e95d9f\" }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.24696914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.248547354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.249376511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.250102954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.25070514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.300906344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.301913584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":16,"elapsed":"2.541876526s"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:16.302632552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.303229989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:16.361713181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 06:58:14.573112433 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.362537498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T06:58:14.573112433Z\",\"pruned\":[]}","iteration":3,"elapsed":"1.776545014s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.581266081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.868588637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.903917664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.904961867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.911560082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:16.957285316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.958304751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":3,"elapsed":"2.499853144s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.959478712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"154fb076-7fb3-4c0e-8ee7-d22348bbf297","remote_addr":"[::1]:37034","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.960208963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"154fb076-7fb3-4c0e-8ee7-d22348bbf297","remote_addr":"[::1]:37034","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:16.96196998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.962781632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":3,"elapsed":"2.436274672s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.963612573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26113","req_id":"d878df46-916c-465e-99bd-eab087167cf9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   8b af 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.965293271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26113","req_id":"d878df46-916c-465e-99bd-eab087167cf9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 35759,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 35759,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:16.971175002Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26113","req_id":"d878df46-916c-465e-99bd-eab087167cf9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:16.971720235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5461","req_id":"1576b467-68a7-43c4-80a8-3446eb465f8b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   5f 58 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   _X..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:16.972631785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5461","req_id":"1576b467-68a7-43c4-80a8-3446eb465f8b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 24408,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 24408,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:16.977802846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5461","req_id":"1576b467-68a7-43c4-80a8-3446eb465f8b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:16.978422226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"154fb076-7fb3-4c0e-8ee7-d22348bbf297","remote_addr":"[::1]:37034","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":17523,"response_code":204}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:17.009771799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:17.057518628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:17.115323894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.180558345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":3,"elapsed":"1.424515315s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:17.181671265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:17.182441715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.183488344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.184609952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.185549999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.186352603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.186806174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.369595799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"154fb076-7fb3-4c0e-8ee7-d22348bbf297\", \"date\": \"Wed, 04 Mar 2026 07:08:16 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.371151899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":3,"elapsed":"22.451049581s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.371973571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:17.37269783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.389846438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.392304881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.393887665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.414254755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.416081503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.502348256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":3,"elapsed":"2.211609068s"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:17.517177879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.517949472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":3,"elapsed":"1.892529859s"}
{"msg":"resolved sleds for vpc oxide-services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.519468569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vpc_to_sled":"[]"}
{"msg":"sending firewall rules to sled agents","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.52099982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"updated firewall rules on 0 sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:17.522919112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"successfully propagated service firewall rules","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.523579884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","elapsed":"123.539849296s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.524216017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":1,"elapsed":"123.540037189s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.687839738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.847404245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.848400745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.849137177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.849869963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.850560534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.86844268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:17.880895923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"468b0930-3e4d-43cf-a384-b36890c98911","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.882354139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"468b0930-3e4d-43cf-a384-b36890c98911","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:17.885875743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:17.904488009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.199761516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.201734785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.34543222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.412818275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.836381946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.868045713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:18.868938857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:18.869683746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:18.870426151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.871257302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:18.879817703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:18.928984332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"a12812d8-39dc-4172-94e7-081ddb110aa8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":3335143,"response_code":200}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:19.247607314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:19.298967189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:19.343647097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:19.426642693Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:19.430705111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:19.432535657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:19.573152183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:19.580644864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:19.743353787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.064968951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.066002734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.066744257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.067801457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.068506246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.077698042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.314230315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.315221042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.315968096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.321796632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.386297927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.387550394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.38865575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.39365553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.427794713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.51463239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.515612476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.516640618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.543826983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.549117801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.549862109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.550569905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.600299913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.607117202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.61566563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.765379214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.766595369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.767324177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.806071532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.81376928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.814523047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":5,"elapsed":"427.76µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.815198889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:20.815805785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.816352711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":5,"elapsed":"135.61µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.817001189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.817589116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.81859788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.81934309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.820034974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.846589804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.852504942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.856690856Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.864459775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.865399682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.866171796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.875542747Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.881695988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:20.882453863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.883011461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":5,"elapsed":"815.97µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:20.883822862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:20.884423224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:21.200610699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:21.201428784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:21.202120117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":3,"elapsed":"6.010139561s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:21.301285127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:21.707432519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:21.711040465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:21.712661384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:21.737661386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:21.782411723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"468b0930-3e4d-43cf-a384-b36890c98911","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":3902296,"response_code":500}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:26.367935561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:26.462898565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:26.87498851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:26.886295045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:26.887551721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:26.91262042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:26.935258751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:49585"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:26.951157297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:26.961395772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:26.965789288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:26.967893998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.004781469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.018579293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.072199237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.104393036Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.133104249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.134043695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.135059202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.14105513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.147221257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:40387","req_id":"68114139-1f1b-4fc1-b836-7ae1a317777b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   4b f8 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   K............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.148307925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:40387","req_id":"68114139-1f1b-4fc1-b836-7ae1a317777b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 19448,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 19448,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:27.153603262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:40387","req_id":"68114139-1f1b-4fc1-b836-7ae1a317777b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.154147102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"468b0930-3e4d-43cf-a384-b36890c98911\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:21 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 807.958935ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:27.154993092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"468b0930-3e4d-43cf-a384-b36890c98911\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"468b0930-3e4d-43cf-a384-b36890c98911\" }"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.332124089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.332959298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":3,"elapsed":"11.735760635s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.336503276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.690631474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.691791558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.692966211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.69794528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:27.739752543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.741249766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":3,"elapsed":"13.279118972s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.755645267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"6da32e9e-f75d-425f-b1a5-e00047486571","remote_addr":"[::1]:49585","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.756581737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"6da32e9e-f75d-425f-b1a5-e00047486571","remote_addr":"[::1]:49585","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.757290865Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54221","req_id":"58767a12-38e4-4303-b6ce-0af74215a148","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   57 a1 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   W............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.761896512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54221","req_id":"58767a12-38e4-4303-b6ce-0af74215a148","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 22433,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 22433,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:27.768298167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54221","req_id":"58767a12-38e4-4303-b6ce-0af74215a148","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.76890382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56261","req_id":"995e5ce7-6322-471b-a3c8-09491c93fbe5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   3e 89 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   >............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.769824638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56261","req_id":"995e5ce7-6322-471b-a3c8-09491c93fbe5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 16009,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 16009,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:27.783819062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56261","req_id":"995e5ce7-6322-471b-a3c8-09491c93fbe5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:27.785406505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.78640115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":3,"elapsed":"13.363893531s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.787365515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61559","req_id":"bcd00822-1ca7-4af9-ba21-643683fb1f83","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   cc 51 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .Q..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.788881887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61559","req_id":"bcd00822-1ca7-4af9-ba21-643683fb1f83","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 52305,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 52305,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:27.796334539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61559","req_id":"bcd00822-1ca7-4af9-ba21-643683fb1f83","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:27.796891605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"6da32e9e-f75d-425f-b1a5-e00047486571","remote_addr":"[::1]:49585","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":39913,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.797933344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.996737045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:27.998210531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:27.999523328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.00936743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:28.089697391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.093408943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.28995539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.305985367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.308026601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.374619911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.3754551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":3,"elapsed":"12.571530984s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:28.37615983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3355","req_id":"1a20ea4d-6d93-44f7-bd2b-147871a3278f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   2b e2 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   +............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.377088243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3355","req_id":"1a20ea4d-6d93-44f7-bd2b-147871a3278f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 11234,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 11234,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:28.382055188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3355","req_id":"1a20ea4d-6d93-44f7-bd2b-147871a3278f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.683369924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6da32e9e-f75d-425f-b1a5-e00047486571\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:08:27 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:28.702106928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:28.703104018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:28.703735181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.704490852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":3,"elapsed":"33.952228897s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.705113238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:28.726854929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.742367418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:28.780484331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:29.218097377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6d9fdd2b-1e17-4a7a-9520-cdb0e7793dde","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:29.240539829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6d9fdd2b-1e17-4a7a-9520-cdb0e7793dde","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:29.260121911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:29.31653913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":5,"elapsed":"8.433359065s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:29.428195649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24363","req_id":"0076fe66-ec1a-495f-ad52-caee86f51a08","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   d7 53 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .S..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:29.454053024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24363","req_id":"0076fe66-ec1a-495f-ad52-caee86f51a08","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55123,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55123,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:29.493033713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24363","req_id":"0076fe66-ec1a-495f-ad52-caee86f51a08","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:29.521681149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:29.606741236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:29.726484335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:29.72770018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:29.728946445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":5,"elapsed":"9.212698697s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.13766301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24143","req_id":"58690f76-40f4-4ee2-84d0-ac6b20abc819","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   56 a0 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   V............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.138788485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24143","req_id":"58690f76-40f4-4ee2-84d0-ac6b20abc819","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 22176,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 22176,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:30.156609669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24143","req_id":"58690f76-40f4-4ee2-84d0-ac6b20abc819","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.273218845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.274125866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.274886226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.275584713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.282253368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.302329391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":5,"elapsed":"9.695144183s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.30422228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.305125213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":17,"elapsed":"11.436251576s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.305785224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.306377971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.307081308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.30792285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.308886233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.313298446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:30.393847323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:30.405345459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.407367705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":5,"elapsed":"9.858424291s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.564539111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.857832325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:30.907368508Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.908251221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:30.916132745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:31.04663843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:31.352028119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:31.451013282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:31.490214499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:31.524575372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:36.659740833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:36.66074168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:36.739982229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.169579373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:37.240240201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.241200769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.318380164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:37.326125667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:59317"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.453456897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.898005237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:37.899566618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.900825258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:37.964205235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.212423357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":9,"elapsed":"11.140068422s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.370231698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":9,"elapsed":"11.41875676s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:38.601501378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"d586c844-4798-4bba-a7cc-986cc0299085","remote_addr":"[::1]:59317","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.602563668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"d586c844-4798-4bba-a7cc-986cc0299085","remote_addr":"[::1]:59317","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:38.807495379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"d586c844-4798-4bba-a7cc-986cc0299085","remote_addr":"[::1]:59317","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":205978,"response_code":200}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.873865436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.875273973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":18,"elapsed":"8.569613056s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.875967531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:38.876570178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:38.877448382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:38.878475261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.879186003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:38.926143945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.063775072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d586c844-4798-4bba-a7cc-986cc0299085\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:08:38 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:39.064927371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.065664334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.076671004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:39.164965124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65227","req_id":"a617b375-4913-495d-b8fd-ae7b8cccd373","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   e7 78 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .x..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.16646377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65227","req_id":"a617b375-4913-495d-b8fd-ae7b8cccd373","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 59256,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 59256,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:39.171863845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65227","req_id":"a617b375-4913-495d-b8fd-ae7b8cccd373","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:39.172414589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50099","req_id":"c000a34b-cf5a-45af-8f38-423fef676a5e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   e6 3f 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .?..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.173332432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50099","req_id":"c000a34b-cf5a-45af-8f38-423fef676a5e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 58943,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 58943,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:39.179566093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50099","req_id":"c000a34b-cf5a-45af-8f38-423fef676a5e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:39.18010343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53238","req_id":"3cd1e509-57b0-4d3a-a5c0-9842941cea1c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   f7 d6 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.18101008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53238","req_id":"3cd1e509-57b0-4d3a-a5c0-9842941cea1c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 63446,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 63446,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:39.269038031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53238","req_id":"3cd1e509-57b0-4d3a-a5c0-9842941cea1c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:39.672033804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:39.673005995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:39.673604614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.674138805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":6,"elapsed":"25.313275146s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.674829667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:39.675724264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.676705112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:39.758313753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.127687349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":5,"elapsed":"19.312580824s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.129190514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.354157097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:40.356014195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.357016025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.397698954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:40.463961437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.464860453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":3,"elapsed":"25.772248965s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:40.517039284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:40.51795377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:40.518705032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.519392247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.56594772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:40.624337487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.189897996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:41.281130151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.331200314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.427298373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:41.504771742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.505960773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":3,"elapsed":"25.765316736s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.506688809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.937858056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:41.93897296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.939364548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:41.997651731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.068767324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:47.110653012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:58490"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.182592841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6a689abf-9c83-43b6-8735-600cd541bad4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.242884956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6a689abf-9c83-43b6-8735-600cd541bad4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.260894535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.261824902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.262672225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.263414209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.264101524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.314478988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.322941054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.323713578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.324768704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.325508163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.326195107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.331164978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.363439507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Err(reqwest::Error { kind: Request, url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", source: TimedOut })"}
{"msg":"failed to contact nexus, will retry in 815.631557ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:47.364297592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Communication Error: error sending request for url (http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229)"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.468942345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6a689abf-9c83-43b6-8735-600cd541bad4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.506964179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6a689abf-9c83-43b6-8735-600cd541bad4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.545204257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6a689abf-9c83-43b6-8735-600cd541bad4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.643747703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.739388022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:47.742580513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.743486793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.774002069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.774982586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":19,"elapsed":"8.900012697s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.775954767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.776806529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.777538532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.778395674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.779570357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.782526658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request handling cancelled (client disconnected)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:47.974271409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6d9fdd2b-1e17-4a7a-9520-cdb0e7793dde","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":18755933}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.975804154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"0cb6ef55-f4bb-4eb3-b020-95cc2dc036c7","remote_addr":"[::1]:59317","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.976589904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"0cb6ef55-f4bb-4eb3-b020-95cc2dc036c7","remote_addr":"[::1]:59317","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:47.978683652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"502c7bb6-dba0-4bf9-83a1-0f8ae44eff6f","remote_addr":"[::1]:58490","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:47.979530394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"502c7bb6-dba0-4bf9-83a1-0f8ae44eff6f","remote_addr":"[::1]:58490","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:48.164706529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"0cb6ef55-f4bb-4eb3-b020-95cc2dc036c7","remote_addr":"[::1]:59317","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":189208,"response_code":204}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:48.165752056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"502c7bb6-dba0-4bf9-83a1-0f8ae44eff6f","remote_addr":"[::1]:58490","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":186901,"response_code":204}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:48.336836123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.337796711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.385286389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.562864524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.564051572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.575076278Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"0cb6ef55-f4bb-4eb3-b020-95cc2dc036c7\", \"date\": \"Wed, 04 Mar 2026 07:08:48 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.57670871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":4,"elapsed":"31.213992434s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.577421445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:48.578339338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:48.578887146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:48.581172746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:48.581957444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.582651222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.651847983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.681364334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"502c7bb6-dba0-4bf9-83a1-0f8ae44eff6f\", \"date\": \"Wed, 04 Mar 2026 07:08:48 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.72884763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:48.729802136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.730435764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":3,"elapsed":"32.923059382s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.734606087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.924502096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:48.925495899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.926244596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:48.965322989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.004004594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.305678163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:49.306741234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.307471034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.363089346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:49.397884142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:49.398792466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:49.399555962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.400248998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.430360603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.438859442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.561429055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:49.563119122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.56441694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.584042608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:49.597647929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6a689abf-9c83-43b6-8735-600cd541bad4","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":2414783,"response_code":200}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:49.677958482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:40074"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:49.844957877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.073061499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:50.074101334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:50.074843098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:50.075577266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.076386824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.087637508Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.186995401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":10,"elapsed":"2.863930572s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.257008854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.364962689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.367547834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":10,"elapsed":"3.106216271s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.370817808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.372743653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":20,"elapsed":"2.596238883s"}
{"msg":"request completed after handler was already cancelled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:50.401300199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6d9fdd2b-1e17-4a7a-9520-cdb0e7793dde","remote_addr":"127.0.0.1:47529","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","response_code":500}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:50.438648806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"2ed18431-e7b3-4ad8-95c1-34e8a693ed65","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.439786826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"2ed18431-e7b3-4ad8-95c1-34e8a693ed65","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:50.442358343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.581427206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.84512449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:50.846502567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.847237136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.855630927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:50.900890175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.901971703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":5,"elapsed":"30.049194551s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:50.974033855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.061010909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.10636703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.169733671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.251992384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.287591757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.28897314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.297530456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.404210722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.458785592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.459584899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":6,"elapsed":"11.623385ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.461205488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:51.4627422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.463315729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":6,"elapsed":"255.657µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.464593016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.465220773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.465925392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.466672987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.467846077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.497823164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.504041003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.504733328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.505441134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.506166845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.506859711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.519131774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.530720571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:51.531497303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.555284236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":6,"elapsed":"1.012241ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.556217039Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.556813053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.558011453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63235","req_id":"3b877bfb-0e24-41d6-8ede-3aab4230c1f0","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   30 41 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   0A..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.558988153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63235","req_id":"3b877bfb-0e24-41d6-8ede-3aab4230c1f0","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 12353,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 12353,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:51.581464362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63235","req_id":"3b877bfb-0e24-41d6-8ede-3aab4230c1f0","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.582078081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59908","req_id":"91789e31-b342-4cb6-8207-84b4a2502c32","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   69 5a 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   iZ..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.582995814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59908","req_id":"91789e31-b342-4cb6-8207-84b4a2502c32","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26970,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26970,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:51.588226472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59908","req_id":"91789e31-b342-4cb6-8207-84b4a2502c32","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.588734732Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63854","req_id":"ae389316-ae6e-43a9-8376-79a4d99b141f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   74 08 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   t............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.589639899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63854","req_id":"ae389316-ae6e-43a9-8376-79a4d99b141f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 29704,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 29704,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:51.595199842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63854","req_id":"ae389316-ae6e-43a9-8376-79a4d99b141f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:51.733057689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.75712567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:51.78234134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:56.03116311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.032504945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:56.038383881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:52314"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.077900725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":21,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.103564946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.10703084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.108869582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.109916451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.122053346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.311854417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":6,"elapsed":"4.927963211s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:56.327870005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"2ed18431-e7b3-4ad8-95c1-34e8a693ed65","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":5888916,"response_code":500}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.331552889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.338111796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.538331619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.539477324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.540215851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.659443006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.750910087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.869481319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"10f994eb-bb8f-47b2-9cb6-0549bca7b2ed","remote_addr":"[::1]:52314","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.870382699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"10f994eb-bb8f-47b2-9cb6-0549bca7b2ed","remote_addr":"[::1]:52314","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.871919411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10603","req_id":"d7df8845-5899-4c41-b50b-7fd025cd1cc5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   08 70 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .p..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.872911331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10603","req_id":"d7df8845-5899-4c41-b50b-7fd025cd1cc5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 2160,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 2160,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:56.878915336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10603","req_id":"d7df8845-5899-4c41-b50b-7fd025cd1cc5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:56.879589955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30051","req_id":"04e59286-c921-4d5e-bf19-28ff4fe27075","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   77 74 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   wt..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.880636955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30051","req_id":"04e59286-c921-4d5e-bf19-28ff4fe27075","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 30580,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 30580,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:56.885814698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30051","req_id":"04e59286-c921-4d5e-bf19-28ff4fe27075","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:56.886336324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2ed18431-e7b3-4ad8-95c1-34e8a693ed65\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:56 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 694.254525ms","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:56.957775418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2ed18431-e7b3-4ad8-95c1-34e8a693ed65\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2ed18431-e7b3-4ad8-95c1-34e8a693ed65\" }"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:57.070078201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"10f994eb-bb8f-47b2-9cb6-0549bca7b2ed","remote_addr":"[::1]:52314","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":201015,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.090583897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.305909832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.306903726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.307636201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.314103375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.381360383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.627911734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.629317797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.630059159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.64384032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:57.668948547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.671318324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:57.672052722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.67272646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":7,"elapsed":"17.998788314s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.673427583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.67401008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.691086866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.72030218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:57.767659194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:57.777221316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.778078218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":6,"elapsed":"6.52606856s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.782059885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.787476583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.789845639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64255","req_id":"5fae6381-4614-44be-840d-c4704ab77325","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   ab 6b 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .k..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.792171328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64255","req_id":"5fae6381-4614-44be-840d-c4704ab77325","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43883,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43883,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:57.799515173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64255","req_id":"5fae6381-4614-44be-840d-c4704ab77325","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.828901346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10f994eb-bb8f-47b2-9cb6-0549bca7b2ed\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:08:57 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.830326938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.831069943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.84940588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.86193281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.863541915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:57.864471331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.86553799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:57.918484867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:58.280738172Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:58.28285692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:58.284362891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.286412141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.295037521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.336805697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":6,"elapsed":"6.876214599s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:58.3558548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"53ddf7a3-711d-4c6f-a742-9e47df45c2aa","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.36474704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"53ddf7a3-711d-4c6f-a742-9e47df45c2aa","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:58.415946637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:58.416757688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.417459091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":6,"elapsed":"7.394133919s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:58.466783483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:37824","req_id":"38f66b11-55d3-41a4-a303-42b2c2ccb0ff","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   a9 d7 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.468336468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:37824","req_id":"38f66b11-55d3-41a4-a303-42b2c2ccb0ff","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43479,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43479,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:58.473722304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:37824","req_id":"38f66b11-55d3-41a4-a303-42b2c2ccb0ff","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:58.542310266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.54710584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.548146858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":21,"elapsed":"2.467857138s"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:08:58.651766385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.652802232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":6,"elapsed":"7.064438868s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:58.853410817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:59.074414148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:08:59.50912047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"53ddf7a3-711d-4c6f-a742-9e47df45c2aa","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":1153289,"response_code":500}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:59.512666613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:59.51358614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:59.687189961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:59.739629551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:59.740558536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:08:59.741316671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:59.742301537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:59.758949011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:08:59.876231823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53ddf7a3-711d-4c6f-a742-9e47df45c2aa\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:59 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 1.071587378s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:08:59.89276003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53ddf7a3-711d-4c6f-a742-9e47df45c2aa\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:08:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53ddf7a3-711d-4c6f-a742-9e47df45c2aa\" }"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:00.018370332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:00.181686632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:00.644030655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:00.665466438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":6,"elapsed":"9.161675271s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.180650406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.201548582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.203381021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":22,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:01.204198575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:01.20496655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:01.205829033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.206574504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.21285638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:01.317000679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"8646e711-8329-4303-9c05-78479129b5a2","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.317945566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"8646e711-8329-4303-9c05-78479129b5a2","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:01.437616823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:01.587682315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:01.589644613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:01.590687063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.591358537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":8,"elapsed":"3.916664288s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:01.756306096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.034084978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.035167478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.036323463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.037152801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.037765558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.047371235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.059848747Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.230083788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.231085477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.232133498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.261191931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:02.290459118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"8646e711-8329-4303-9c05-78479129b5a2","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":973524,"response_code":500}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.338797181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.339598833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.340627877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.341644326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.342338164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.357012358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.363473831Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.393892555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.394812082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.395864522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.396952974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.41892887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.441892828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8646e711-8329-4303-9c05-78479129b5a2\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:02 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 1.866741697s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:02.442949517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8646e711-8329-4303-9c05-78479129b5a2\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8646e711-8329-4303-9c05-78479129b5a2\" }"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.555724357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.556851916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":22,"elapsed":"1.352319388s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.623740442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.76942833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:02.770453166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.771169709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.808442826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:02.819136334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.889593829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:02.964276695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.251162013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:03.252200345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.252921838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.258600857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.290708666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.523770827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:03.524761965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.526024212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:03.531740966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.02377867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:07.065933022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:48534"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.099295502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"60644da4-b7e2-4480-9570-fb39ccc7d77d","remote_addr":"[::1]:52314","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.100913846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"60644da4-b7e2-4480-9570-fb39ccc7d77d","remote_addr":"[::1]:52314","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.124205395Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.125764612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.221923712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":23,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.222852256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.22360924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.224635087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.225353173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.230378815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:07.303937618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"60644da4-b7e2-4480-9570-fb39ccc7d77d","remote_addr":"[::1]:52314","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":204715,"response_code":204}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.379118784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":11,"elapsed":"5.015491459s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.380107709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":11,"elapsed":"5.041887393s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.417309474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"b59dadb4-6c9b-4f66-ab61-9a22a93b2b19","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.418231746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"b59dadb4-6c9b-4f66-ab61-9a22a93b2b19","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.419862565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"903d02c9-7d14-4177-98c5-8c003305d8e1","remote_addr":"[::1]:48534","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.420580882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"903d02c9-7d14-4177-98c5-8c003305d8e1","remote_addr":"[::1]:48534","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:07.451353879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.50892968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"60644da4-b7e2-4480-9570-fb39ccc7d77d\", \"date\": \"Wed, 04 Mar 2026 07:09:07 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.509955707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":5,"elapsed":"18.932537408s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.51060673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:07.511217554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.511752225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.51246926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.513196263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.513882015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.522333651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:07.56059922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"903d02c9-7d14-4177-98c5-8c003305d8e1","remote_addr":"[::1]:48534","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":140776,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.580639872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.765491433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.766913276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.768097198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.815909636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.898684464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61797","req_id":"fdcce362-b76e-4bcb-8a61-fc1668082e30","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   0b 5f 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ._..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.951738874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61797","req_id":"fdcce362-b76e-4bcb-8a61-fc1668082e30","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 2911,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 2911,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:07.962790564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61797","req_id":"fdcce362-b76e-4bcb-8a61-fc1668082e30","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.963590914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52503","req_id":"b509dbb7-b260-4780-94c5-689fed801de7","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   73 ab 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   s............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.964692972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52503","req_id":"b509dbb7-b260-4780-94c5-689fed801de7","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 29611,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 29611,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:07.9722795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52503","req_id":"b509dbb7-b260-4780-94c5-689fed801de7","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:07.972825775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63327","req_id":"a4e8f007-efbe-4110-9037-17bae876c5e1","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   4c 61 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   La..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:07.974483487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63327","req_id":"a4e8f007-efbe-4110-9037-17bae876c5e1","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 19553,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 19553,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:07.982553194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63327","req_id":"a4e8f007-efbe-4110-9037-17bae876c5e1","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.053405272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"903d02c9-7d14-4177-98c5-8c003305d8e1\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:09:07 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:08.05475355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:08.055486566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:08.056098602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.056684365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":4,"elapsed":"39.350434569s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.057289537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:08.057863928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.058544229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.06254282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.192253787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.326090087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:08.327105624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:08.327853589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:08.32859887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.329302227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.425594591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.462170935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.463742877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":23,"elapsed":"1.24063866s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.650981899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.924713632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:08.926342437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.927163819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:08.99283653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:12.357244342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.358209999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.361725251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.36264104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.372910446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:12.388198219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:41569"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.403680027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":24,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.404555576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.406183779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.407032745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.407685873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.415651293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.493116204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"8dc99a74-77cd-40f7-ae27-1cc8a43e26f1","remote_addr":"[::1]:41569","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.495981204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"8dc99a74-77cd-40f7-ae27-1cc8a43e26f1","remote_addr":"[::1]:41569","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:12.695597008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"8dc99a74-77cd-40f7-ae27-1cc8a43e26f1","remote_addr":"[::1]:41569","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":202580,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.717628544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.718907294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.71960989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.752631553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.868489226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8dc99a74-77cd-40f7-ae27-1cc8a43e26f1\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:09:12 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:12.869922222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.870656731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.876588882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:12.956895127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.179110866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.180115621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.180996531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.187849752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.194353098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.195189119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.195926363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.196618618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.211667546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.236025724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.237187912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.23796745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.23894416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.256417624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.30613181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.309249632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":24,"elapsed":"905.554466ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.348699984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.403352769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.550326371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.770265147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.774001779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.776290915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.780590274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.782622981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.788208364Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.830739865Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.832373279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.841533662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.935876545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.936908975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:13.937695107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.938386069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.959558249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:13.975948692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:14.097991168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:14.099524203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:14.100431966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:14.108742401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:14.116820585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:14.118098182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:14.138920277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:14.282583464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.675947624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:17.784297075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"b59dadb4-6c9b-4f66-ab61-9a22a93b2b19","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":10366962,"response_code":500}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:17.848771316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:60185"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:17.853864872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c64ff51f-da39-4754-9e72-cd0ecac76ac8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.912300811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c64ff51f-da39-4754-9e72-cd0ecac76ac8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:17.915518341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"643edbc5-b9bf-4614-aceb-750845179d79","remote_addr":"[::1]:41569","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.91638366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"643edbc5-b9bf-4614-aceb-750845179d79","remote_addr":"[::1]:41569","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.940424006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.941318673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:17.942004785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.942733412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.948176302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.980668344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:17.981979659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.982641173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":4,"elapsed":"530.263µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.9833298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:17.983948569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.984838426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.985718043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:17.986595446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.987337229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:17.992839197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.008613596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.00984406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.013507155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:18.014992435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.016615418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":4,"elapsed":"3.226999ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.020943825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:18.02412372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 06:59:18.023595331 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.0268475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.028740159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.087347087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.101167636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.106436059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.108246515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.109677437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.111097237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.121029122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.151938882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:18.228930704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.230794696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":4,"elapsed":"78.814432ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.233038833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.234357302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.235862912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.24532773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.253016601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.255484933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.258021771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.26978819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.279456793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.281422156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.284400391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.292887517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.303812555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.348235714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.43971553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.535529094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.586130673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.586942515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.587530262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.588104803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.588650958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.589167534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.589750171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.590488037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.591245852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.59253395Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.598240905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.604423454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:18.605226208Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.605794837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.608610029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:18.609377092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.610226479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.612065812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.612822545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.613517195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.622226432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.623267189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.62388213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.624589275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.625321609Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.62600681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.652648092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.653574552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.654189784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.654893271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.655613601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.656293963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.656855628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.65756128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":25,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.66098541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.662110976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.662901195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.689909887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.706966943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.707874436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b59dadb4-6c9b-4f66-ab61-9a22a93b2b19\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:17 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 1.926230839s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:18.708736538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b59dadb4-6c9b-4f66-ab61-9a22a93b2b19\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b59dadb4-6c9b-4f66-ab61-9a22a93b2b19\" }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.866685007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c64ff51f-da39-4754-9e72-cd0ecac76ac8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.867928997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c64ff51f-da39-4754-9e72-cd0ecac76ac8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.877002849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c64ff51f-da39-4754-9e72-cd0ecac76ac8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:18.940400662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"643edbc5-b9bf-4614-aceb-750845179d79","remote_addr":"[::1]:41569","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":1026072,"response_code":204}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.970486214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"5ba02c04-dab6-4115-b98f-ed5177976d9b","remote_addr":"[::1]:60185","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.973945715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"5ba02c04-dab6-4115-b98f-ed5177976d9b","remote_addr":"[::1]:60185","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.981067911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.982093648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:18.982898647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.984033791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:18.993842001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.015480677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"643edbc5-b9bf-4614-aceb-750845179d79\", \"date\": \"Wed, 04 Mar 2026 07:09:18 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.016702484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":6,"elapsed":"11.505958841s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.017500068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:19.01850337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.019057671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.019858412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.020947034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.021726091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.024672974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:19.176357141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"5ba02c04-dab6-4115-b98f-ed5177976d9b","remote_addr":"[::1]:60185","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":205810,"response_code":200}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.448453096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21549","req_id":"86c9895f-9587-4b2a-a79f-5fd08e6a7017","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   9d 27 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   .'..........._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.449635655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21549","req_id":"86c9895f-9587-4b2a-a79f-5fd08e6a7017","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 40231,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 40231,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.455858163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21549","req_id":"86c9895f-9587-4b2a-a79f-5fd08e6a7017","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.456504437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45785","req_id":"ef7320b6-ae4b-48f7-82c8-b2bffa680164","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   36 65 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   6e..........._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.457421308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45785","req_id":"ef7320b6-ae4b-48f7-82c8-b2bffa680164","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 13925,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 13925,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.463146919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:45785","req_id":"ef7320b6-ae4b-48f7-82c8-b2bffa680164","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.463782181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:4850","req_id":"14cf502a-53fa-417d-942b-2932955c875d","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   f4 c1 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.4647231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:4850","req_id":"14cf502a-53fa-417d-942b-2932955c875d","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62657,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62657,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.470785951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:4850","req_id":"14cf502a-53fa-417d-942b-2932955c875d","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.471448717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:41368","req_id":"8b42fa7a-83e3-4179-ad29-df28b0a97892","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   1d 43 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .C..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.472317844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:41368","req_id":"8b42fa7a-83e3-4179-ad29-df28b0a97892","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 7491,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 7491,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.477479475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:41368","req_id":"8b42fa7a-83e3-4179-ad29-df28b0a97892","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.493119598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ba02c04-dab6-4115-b98f-ed5177976d9b\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:09:19 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:19.494366995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:19.495007467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:19.495722718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.496334693Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":5,"elapsed":"11.439896998s"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:19.685223342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.6862949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":4,"elapsed":"1.695657076s"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:19.713576133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.714475348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":4,"elapsed":"1.689915201s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.715356348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.716051749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.716782461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.717512971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":4,"elapsed":"1.115454064s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.718412007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.719157478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.72098163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.721724316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.722307844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.723028586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61718","req_id":"0e54e315-f031-4852-828f-012be4e989aa","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   de 2d 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .-..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.723944545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61718","req_id":"0e54e315-f031-4852-828f-012be4e989aa","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56877,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56877,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.729992427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61718","req_id":"0e54e315-f031-4852-828f-012be4e989aa","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.730562188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55526","req_id":"ef9dbd40-f358-44c9-b318-e616711eed77","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   ac 0f 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.744422556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55526","req_id":"ef9dbd40-f358-44c9-b318-e616711eed77","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 44047,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 44047,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.750093549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55526","req_id":"ef9dbd40-f358-44c9-b318-e616711eed77","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.750632269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51754","req_id":"3ba4193f-a59b-4a34-be4f-bf1c3ebddb40","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   ce 33 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .3..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.751539391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51754","req_id":"3ba4193f-a59b-4a34-be4f-bf1c3ebddb40","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 52787,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 52787,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:19.756941961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51754","req_id":"3ba4193f-a59b-4a34-be4f-bf1c3ebddb40","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.75745506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.75819014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:19.759969494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.760779753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:19.761402359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:20.010534817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 06:59:18.023595331 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.02419128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T06:59:18.023595331Z\",\"pruned\":[]}","iteration":4,"elapsed":"1.990272936s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.067067195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.3456935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:20.346678265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.347407885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.389324314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:20.396005283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:20.396819049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:20.397798735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.398363296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":9,"elapsed":"7.679568726s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.399312391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.508209179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:20.509280727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:20.510024244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:20.510764544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.511459845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.516127536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.523058221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.801546351Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:20.814774413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.815696264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.83013975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:20.876843317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.191118083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:21.192055334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.192783801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.204089345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.228977906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.22977583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.233133729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:21.244974236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.245962348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.360808933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.559935755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.712096589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:21.713185141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.713948317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.720212458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.736588673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":4,"elapsed":"3.483469749s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:21.737429022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.097577085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.102036342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.103677131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.114559263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:22.125445845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c64ff51f-da39-4754-9e72-cd0ecac76ac8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":4271780,"response_code":200}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:22.158761202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.176739899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":4,"elapsed":"3.872417471s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.38108209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.630924037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.649780447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"03fbd2ac-851f-488c-b073-c3ca1f724d06","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.6508291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"03fbd2ac-851f-488c-b073-c3ca1f724d06","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.651669399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.652266054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.653072686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.729352574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.735062705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.739379729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.740400636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":7,"elapsed":"333.201µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.741068964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:22.741674135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.742211322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":7,"elapsed":"133.485µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.74285475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.743436546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.744131606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.745164077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.746138222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.780438244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.788263255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.789044818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.790061948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.79080317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.79161895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.833691279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.853512696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:22.854378987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.855500153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":7,"elapsed":"1.01229ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:22.856583876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:22.857186402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:22.921839078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.354014574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:23.379379281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.382187809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":4,"elapsed":"5.148601268s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:23.487805013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3282","req_id":"582428e3-72e0-4a7e-bc6d-40a7b444e555","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   c4 0b 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.48895121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3282","req_id":"582428e3-72e0-4a7e-bc6d-40a7b444e555","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50187,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50187,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:23.495066735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3282","req_id":"582428e3-72e0-4a7e-bc6d-40a7b444e555","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.679276121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.903150262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:23.904378291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:23.905262467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:23.906064801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.906799289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:23.915833252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:24.004404952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.007940354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":7,"elapsed":"1.153292773s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.130918409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.349932258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:24.351105579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.351837612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.359401044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.429056726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.691387096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:24.72974598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:24.730755735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:24.731516917Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.732617583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.738054441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:24.975802608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:24.976607427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:24.97734934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.008406592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.062203237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.063490393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":25,"elapsed":"6.405117572s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.064189933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":26,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.0988999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.099833203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.100716017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.101428712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.111019811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.122704217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":12,"elapsed":"6.478540375s"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.353860432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.385157862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.416874043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":12,"elapsed":"6.795461938s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.460563353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.553704649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.554713533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.555496598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.56115188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:25.888678455Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:29531","req_id":"a7b86f2f-474f-4022-af71-cefcf92f4c0a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   1b 39 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .9..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:25.889963808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:29531","req_id":"a7b86f2f-474f-4022-af71-cefcf92f4c0a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6969,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6969,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:25.918678478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:29531","req_id":"a7b86f2f-474f-4022-af71-cefcf92f4c0a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.226854942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":7,"elapsed":"3.491560918s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.275005232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.393262781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:26.435462664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.501623334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.529511142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:26.589695232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:26.590712713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.59142603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":7,"elapsed":"5.35715897s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:26.592239355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58151","req_id":"d96393ed-bdc1-4830-818e-5f3989ac7c41","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   f1 b0 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.593156677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58151","req_id":"d96393ed-bdc1-4830-818e-5f3989ac7c41","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 61872,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 61872,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:26.598964763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58151","req_id":"d96393ed-bdc1-4830-818e-5f3989ac7c41","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:26.624936334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.62589615Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":4,"elapsed":"8.684970819s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.626989031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.824071783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:26.825066118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.825800186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.854361481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:26.939439403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.315765868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:27.316888398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.317620883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.322842834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:27.402619096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.403526548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":4,"elapsed":"9.123515204s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.404304303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":7,"elapsed":"4.664308361s"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:27.407068413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.407919042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":7,"elapsed":"4.619603484s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:27.432361337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:27.433489718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.434217052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":7,"elapsed":"4.782043151s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.815862842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:27.816660386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.817742656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:27.853795289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:28.069317434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:28.070446927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":4,"elapsed":"9.968061252s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:28.274906311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:28.393343918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:28.439110915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:28.440006363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:28.452715094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:29.160678437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:29.315379336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:29.795187555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:29.796178894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:29.840809885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:30.215008775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.255050441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":4,"elapsed":"12.271783575s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:30.315914882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:30.341221063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:30.342152623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.342869006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.376145876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:30.412094353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52249","req_id":"2e9720d3-3940-4c43-ba06-7030fdea4e8b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   b3 41 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .A..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.438046586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52249","req_id":"2e9720d3-3940-4c43-ba06-7030fdea4e8b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45889,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45889,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:30.443841986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52249","req_id":"2e9720d3-3940-4c43-ba06-7030fdea4e8b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:30.444401598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50105","req_id":"41e93f27-d271-405e-aaa3-93f5ddb7aa8c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   bc 7f 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.445316595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50105","req_id":"41e93f27-d271-405e-aaa3-93f5ddb7aa8c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 48255,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 48255,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:30.470152521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50105","req_id":"41e93f27-d271-405e-aaa3-93f5ddb7aa8c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:30.470775759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50884","req_id":"ec91e3cb-e874-43ae-b7de-a2d7d4652edb","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   15 39 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .9..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.471699263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50884","req_id":"ec91e3cb-e874-43ae-b7de-a2d7d4652edb","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 5433,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 5433,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:30.477982442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50884","req_id":"ec91e3cb-e874-43ae-b7de-a2d7d4652edb","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:30.487972984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.492895581Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":4,"elapsed":"11.90421332s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.683586118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:30.72799595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:31.636033797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"03fbd2ac-851f-488c-b073-c3ca1f724d06","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":8986380,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:31.665627062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03fbd2ac-851f-488c-b073-c3ca1f724d06\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:31 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 1.482060941s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:31.666669993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03fbd2ac-851f-488c-b073-c3ca1f724d06\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03fbd2ac-851f-488c-b073-c3ca1f724d06\" }"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:31.985050884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.285130226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:32.286801726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.288039334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.296970452Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:32.367336064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:32.386681055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:32.387829726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.389047755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":10,"elapsed":"4.573724635s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.541432574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.567755563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":26,"elapsed":"7.504158058s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.568592746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":27,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:32.569202136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:32.56991888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:32.570771333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.571478608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:32.604699988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.174653953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.175401789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.182375038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"49ddd1d6-61f6-44c3-b56a-8377b2e56bdf","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.18325717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"49ddd1d6-61f6-44c3-b56a-8377b2e56bdf","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:33.334175376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.57617623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.577274832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":4,"elapsed":"14.972573917s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.738266404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.779295361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.831827073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.87369146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.874789651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.899995141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.926862712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.927654234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.956050361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:33.965203861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.966708148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:33.976510587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.14390708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.410770222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:34.416156169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.41773751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.426724649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.497404355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.676214898Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:34.677333239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.678069671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.68801323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.834368664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:34.835175276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:34.83591766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:34.836649895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.837343913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.865626825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.950596853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":13,"elapsed":"1.211742541s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:34.966824061Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":13,"elapsed":"1.039796733s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:35.134280083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:35.443506292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:35.444600284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:35.445332749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:35.451563925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:39.530402464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.531391328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:39.534492577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:39.535373627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:39.53612524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.536948255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.546616587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read target blueprint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:39.607925932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: no target blueprint set"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.609398677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read target blueprint: Internal Error: no target blueprint set\"}","iteration":3,"elapsed":"4.773954317s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:39.610408112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"49ddd1d6-61f6-44c3-b56a-8377b2e56bdf","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":6428584,"response_code":500}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.61247677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.841910033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:39.843723425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.844869451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:39.850828637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:39.966153742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:36683"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.014117642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.014914194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.015639004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.027246627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.034230828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49ddd1d6-61f6-44c3-b56a-8377b2e56bdf\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:39 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 2.147960251s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:40.035297407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49ddd1d6-61f6-44c3-b56a-8377b2e56bdf\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"49ddd1d6-61f6-44c3-b56a-8377b2e56bdf\" }"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.110249267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.111270957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":27,"elapsed":"7.542923858s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.111958242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":28,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.112678051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.113686233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.114437375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.115708179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.181597611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.237609135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.340217764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.341616051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.34235077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.3482164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.39346628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"f7a41333-0845-44d9-929c-b10f6ef8675c","remote_addr":"[::1]:36683","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.394361718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"f7a41333-0845-44d9-929c-b10f6ef8675c","remote_addr":"[::1]:36683","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:40.39590853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"f7a41333-0845-44d9-929c-b10f6ef8675c","remote_addr":"[::1]:36683","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":2577,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.46212932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7a41333-0845-44d9-929c-b10f6ef8675c\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:09:40 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.463762544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.464493786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.470576938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.568425563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.923303229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:40.933324062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.935180258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:40.944891636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:43.800136987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:43.884461733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:43.95427607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:43.991538737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.017597933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.018654381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":28,"elapsed":"3.906226937s"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.02321074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.024112882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.123901661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:33717"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.150745725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"fcded2dc-ffd0-4492-83e3-da90d0743971","remote_addr":"[::1]:36683","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.152322928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"fcded2dc-ffd0-4492-83e3-da90d0743971","remote_addr":"[::1]:36683","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.156963184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.159385555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.162982529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.164983593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.16629642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.17528406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.222264608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"fcded2dc-ffd0-4492-83e3-da90d0743971","remote_addr":"[::1]:36683","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":71160,"response_code":204}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.250533783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"1140a023-5540-496c-a50a-aabfc9f102bb","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.251432999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"1140a023-5540-496c-a50a-aabfc9f102bb","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.252140123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"9bb2f5f3-e5fe-449d-834d-18fd865394cb","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.252820334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"9bb2f5f3-e5fe-449d-834d-18fd865394cb","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.267300513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"fa276ef3-a750-452b-800d-69755deccc15","remote_addr":"[::1]:33717","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.268423293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"fa276ef3-a750-452b-800d-69755deccc15","remote_addr":"[::1]:33717","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.269133102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56671","req_id":"e645db46-c8ca-4749-b1e0-73ef1c99ad30","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   5f 94 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   _............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.277501222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56671","req_id":"e645db46-c8ca-4749-b1e0-73ef1c99ad30","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 24468,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 24468,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:44.287391926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56671","req_id":"e645db46-c8ca-4749-b1e0-73ef1c99ad30","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.292961928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52858","req_id":"6e766937-2008-4bbe-9e8e-fd9d88bdf75c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   34 bc 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   4............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.294524913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52858","req_id":"6e766937-2008-4bbe-9e8e-fd9d88bdf75c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 13500,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 13500,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:44.301448002Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52858","req_id":"6e766937-2008-4bbe-9e8e-fd9d88bdf75c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.302277961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55766","req_id":"200dec51-9605-4b84-a531-e72f0d3fafdf","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   b0 8d 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.303209852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55766","req_id":"200dec51-9605-4b84-a531-e72f0d3fafdf","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45197,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45197,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:44.308877037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55766","req_id":"200dec51-9605-4b84-a531-e72f0d3fafdf","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.32196955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"1140a023-5540-496c-a50a-aabfc9f102bb","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.323068092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"1140a023-5540-496c-a50a-aabfc9f102bb","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.350772966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"1140a023-5540-496c-a50a-aabfc9f102bb","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.358884767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.364586681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"fcded2dc-ffd0-4492-83e3-da90d0743971\", \"date\": \"Wed, 04 Mar 2026 07:09:44 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.365548912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":7,"elapsed":"25.348918962s"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.383021334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"fa276ef3-a750-452b-800d-69755deccc15","remote_addr":"[::1]:33717","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":115937,"response_code":204}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.387348027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.388164629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.38891046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.389915115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.397912128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.600033167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"fa276ef3-a750-452b-800d-69755deccc15\", \"date\": \"Wed, 04 Mar 2026 07:09:44 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.610783148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.61786841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.625381982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":4,"elapsed":"26.016622978s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.628083347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.65369286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.748827195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.749836289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:44.750612701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.751313332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.751885439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.752601461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:44.868728989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:44.954606209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"1140a023-5540-496c-a50a-aabfc9f102bb","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":646934,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.071675257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.32960625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.355138689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.357009605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.398335522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.518890454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.55580528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.585367062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.626574163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.65215074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.73592359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.737011711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.737778103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.738476239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.749071202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.756011596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.857583515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.858518713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.859535402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.8722729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.903232558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":29,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.904125211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.904979107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:45.905948161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.906653542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:45.91156167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.083833346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.360301144Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:46.361623631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.36236825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.367998622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.62390349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.692983569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.756063892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:46.805117225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":29,"elapsed":"902.09742ms"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:50.734895909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.735750436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:50.786031519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:56106"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.792751626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:50.793538649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:50.794275232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:50.795012736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.795821533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.801094325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.831853796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:50.832630298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:50.833358995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:50.834087201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.834777021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:50.839990235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.014359808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"f464d24a-c454-46cc-89ff-8177a2625974","remote_addr":"[::1]:56106","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.015324043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"f464d24a-c454-46cc-89ff-8177a2625974","remote_addr":"[::1]:56106","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.016287466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":30,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.016951575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.018010629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.018767802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.019656217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.065115062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:51.118328799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"f464d24a-c454-46cc-89ff-8177a2625974","remote_addr":"[::1]:56106","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":104543,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.186038726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f464d24a-c454-46cc-89ff-8177a2625974\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:09:51 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.201346419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.213883488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.221320228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.280518371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.498264082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.511722652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.517544184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.525212874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.543248585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.805243047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:51.806296849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.807021549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:51.81743957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.037132158Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.388793015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.390127275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.390984197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.39623703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.694305638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.69522263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.696065614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.712017626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.747401872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.748283403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.74901713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.780308597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:52.792667703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.793472722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:52.794049217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.79469003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":11,"elapsed":"8.630164577s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.916031313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":14,"elapsed":"2.084000175s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.932565241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":14,"elapsed":"2.139724639s"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.933411282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.954736926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.961260302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.962198115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":8,"elapsed":"356.909µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.962882083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:52.979522302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.980467239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":8,"elapsed":"17.910442ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.981331836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.981936627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.982836304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:52.983842803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.984735466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:52.994660477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.001647765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.002574596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.003360256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.004185375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.004966997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.064641666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.119534628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:53.121038926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.121667314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":8,"elapsed":"1.021208ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.122511069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.123131131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:53.268610996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"9bb2f5f3-e5fe-449d-834d-18fd865394cb","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":9016150,"response_code":500}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.276906471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.278597871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":30,"elapsed":"2.261872415s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.292623948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.293695636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.294629491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.295687692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.330292671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.335981078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.408112307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21183","req_id":"06dd1190-2775-476d-8986-9275c058be5d","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   ee 57 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .W..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.409949486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21183","req_id":"06dd1190-2775-476d-8986-9275c058be5d","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 61015,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 61015,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:53.421818509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21183","req_id":"06dd1190-2775-476d-8986-9275c058be5d","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.422433991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9bb2f5f3-e5fe-449d-834d-18fd865394cb\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:53 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 1.997949195s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:53.423286474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9bb2f5f3-e5fe-449d-834d-18fd865394cb\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9bb2f5f3-e5fe-449d-834d-18fd865394cb\" }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.761660363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":8,"elapsed":"806.993336ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.785245184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.890875745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:53.892224353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.892963371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.898863409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:53.907165498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.144603459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.145618154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.146343936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.147073224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.147764738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.154110271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.247366866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.461759648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.462745506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.463469043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.47203726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:54.498826375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:54.49973028Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.500444389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":8,"elapsed":"1.752726309s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.56774343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:13005","req_id":"49038c47-5eee-4003-ac80-de62b1fafe5c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   31 7b 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   1{..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.573927572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:13005","req_id":"49038c47-5eee-4003-ac80-de62b1fafe5c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 12667,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 12667,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:54.608636207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:13005","req_id":"49038c47-5eee-4003-ac80-de62b1fafe5c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.695806735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63169","req_id":"dda8841d-744c-4e4a-8517-868c19f613c9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   0d cf 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.763925123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63169","req_id":"dda8841d-744c-4e4a-8517-868c19f613c9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 3535,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 3535,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:54.775599149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63169","req_id":"dda8841d-744c-4e4a-8517-868c19f613c9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.821749978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.891801606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:54.893985494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.895680981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.90419958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:54.967554608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":8,"elapsed":"1.98768708s"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:55.113253017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:55.114915779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":8,"elapsed":"1.993247083s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:55.115843312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:55.116679562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:55.117376406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":8,"elapsed":"2.422290478s"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.440683748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.445088757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.446548606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":8,"elapsed":"5.444065593s"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.447407092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.448341006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.462418636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:51504"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.495341092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"df34aad2-5364-471d-9559-3aad0de8a32b","remote_addr":"[::1]:56106","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.496139598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"df34aad2-5364-471d-9559-3aad0de8a32b","remote_addr":"[::1]:56106","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.496739891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.498003049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.49977074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":31,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.500631971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.50150778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.502128583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.50271095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.510056579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.521937535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56365","req_id":"fa3a8f91-1180-4dd4-9f3a-2ccf5e3fa144","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   15 bf 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.523694735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56365","req_id":"fa3a8f91-1180-4dd4-9f3a-2ccf5e3fa144","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 5567,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 5567,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:58.529075913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56365","req_id":"fa3a8f91-1180-4dd4-9f3a-2ccf5e3fa144","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.533493647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51343","req_id":"fbcf3268-1449-4cb5-b35d-53ff04c306d5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   03 60 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .`..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.535792542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51343","req_id":"fbcf3268-1449-4cb5-b35d-53ff04c306d5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 864,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 864,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:58.543153372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51343","req_id":"fbcf3268-1449-4cb5-b35d-53ff04c306d5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.549026987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"df34aad2-5364-471d-9559-3aad0de8a32b","remote_addr":"[::1]:56106","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":53994,"response_code":204}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.585113157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"14ce206d-e771-413d-96e7-0e036bd2cefb","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.585899449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"14ce206d-e771-413d-96e7-0e036bd2cefb","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.589632703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"c820ce48-413f-47d8-a921-f9d97baa269b","remote_addr":"[::1]:51504","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.590400037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"c820ce48-413f-47d8-a921-f9d97baa269b","remote_addr":"[::1]:51504","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.59097012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64171","req_id":"96b7f1d3-b6e2-4d3b-b8ca-22d5d554313d","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   dd e9 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.591740299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64171","req_id":"96b7f1d3-b6e2-4d3b-b8ca-22d5d554313d","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56809,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56809,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:09:58.597372295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64171","req_id":"96b7f1d3-b6e2-4d3b-b8ca-22d5d554313d","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.618819591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.627635109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"df34aad2-5364-471d-9559-3aad0de8a32b\", \"date\": \"Wed, 04 Mar 2026 07:09:58 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.630139263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":8,"elapsed":"14.002145263s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.631415027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.632400514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.633282756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.634728097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.635925435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.637024558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.645662403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:58.729352869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"c820ce48-413f-47d8-a921-f9d97baa269b","remote_addr":"[::1]:51504","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":140127,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.744151771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.890814595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:58.891736877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.892466776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.89905981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:58.991908526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.262618067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.263679295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.26441719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.305843246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.307150202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.317617401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.323657386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.324673224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":31,"elapsed":"824.155921ms"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:59.3285329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"14ce206d-e771-413d-96e7-0e036bd2cefb","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":743693,"response_code":500}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.437547201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.438438932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.439169834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.450821525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.467473778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14ce206d-e771-413d-96e7-0e036bd2cefb\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:59 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 4.329733826s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:09:59.468646076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14ce206d-e771-413d-96e7-0e036bd2cefb\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:09:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"14ce206d-e771-413d-96e7-0e036bd2cefb\" }"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.472571031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c820ce48-413f-47d8-a921-f9d97baa269b\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:09:58 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:59.473627489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:59.474246579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:09:59.474837583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.475420881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":6,"elapsed":"19.460184337s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.603144491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.603942967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.604961219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.605655097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.643875386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.716596767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.884533683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:09:59.906730087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.941316708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:09:59.978707349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:00.084731773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:00.203049712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:03.050587235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.054079742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.059771536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.064142537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.073263342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:03.109343791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:39366"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.227416583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.228344175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.229105377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.23005843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.234968642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.289665044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"b9c5fccd-e566-46ee-aa62-bdc28798a5ad","remote_addr":"[::1]:39366","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.290671413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"b9c5fccd-e566-46ee-aa62-bdc28798a5ad","remote_addr":"[::1]:39366","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:03.350124281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"b9c5fccd-e566-46ee-aa62-bdc28798a5ad","remote_addr":"[::1]:39366","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":60567,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.516043039Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.657957223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.664286633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b9c5fccd-e566-46ee-aa62-bdc28798a5ad\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:10:03 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.665278513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.665986519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.71395556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.772587197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":32,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.773392446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.774141795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.774885392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.775581594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.803587265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.850384408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.851760681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:03.868497712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"9e37a568-237d-476f-937a-bd4a7b0b8748","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:03.870141597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"9e37a568-237d-476f-937a-bd4a7b0b8748","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:03.980750816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.260091901Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.485720157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:04.486859049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.487899375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.492435695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.706123276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.707256367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":32,"elapsed":"933.872096ms"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:04.775751604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:04.776682012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:04.777282645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:04.777818108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":12,"elapsed":"5.338903662s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:05.973324561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:05.974221643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:05.975248282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:05.976011869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:05.976708362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.083968299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.099882817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:06.100600482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:06.101656119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:06.102419685Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.103104826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.108668977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.520856806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.797933773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:06.798890152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.799487488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.85551334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.911929838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.987466941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:06.988651233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.989545309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:06.993652606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.159497087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":15,"elapsed":"1.059714129s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.171711156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":15,"elapsed":"1.19869214s"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:07.628202084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.629500573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:07.641340097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"9e37a568-237d-476f-937a-bd4a7b0b8748","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":3773181,"response_code":500}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:07.664680764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"3946562c-5ffb-49e4-b7ab-1d2806a5a47e","remote_addr":"[::1]:39366","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.665677453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"3946562c-5ffb-49e4-b7ab-1d2806a5a47e","remote_addr":"[::1]:39366","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.667107614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e37a568-237d-476f-937a-bd4a7b0b8748\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:07 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 2.250978066s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:07.668022641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e37a568-237d-476f-937a-bd4a7b0b8748\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e37a568-237d-476f-937a-bd4a7b0b8748\" }"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:07.679557561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"3946562c-5ffb-49e4-b7ab-1d2806a5a47e","remote_addr":"[::1]:39366","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":14974,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.742140666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"3946562c-5ffb-49e4-b7ab-1d2806a5a47e\", \"date\": \"Wed, 04 Mar 2026 07:10:07 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:07.743153978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":9,"elapsed":"9.110789957s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:08.660152923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54670","req_id":"567c628b-3aa1-4f6b-95ef-6dc5006ab212","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   45 93 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   E............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:08.661629767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54670","req_id":"567c628b-3aa1-4f6b-95ef-6dc5006ab212","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 17811,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 17811,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:08.666729315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54670","req_id":"567c628b-3aa1-4f6b-95ef-6dc5006ab212","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:08.667324858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53313","req_id":"eb324a73-110f-4ddf-a971-5c4b6fda5de7","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   9d c1 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:08.691144515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53313","req_id":"eb324a73-110f-4ddf-a971-5c4b6fda5de7","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 40385,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 40385,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:08.70158956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53313","req_id":"eb324a73-110f-4ddf-a971-5c4b6fda5de7","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:08.975655607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":33,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:08.976535615Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:08.977294202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:08.978361692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:08.979574861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:08.991196743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:09.131498334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:09.744470016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:09.745449412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:09.746065405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:09.752952594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.021769085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.022530808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.024498526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:10.02581994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.02653956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.032946895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:10.065696929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54784","req_id":"b6413029-8576-44e1-bc2b-da0a3b00fc23","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   dc 77 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .w..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.068156474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54784","req_id":"b6413029-8576-44e1-bc2b-da0a3b00fc23","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56439,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56439,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:10.077059126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54784","req_id":"b6413029-8576-44e1-bc2b-da0a3b00fc23","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.11586137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.119084842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":33,"elapsed":"1.143848288s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:10.125899246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"000d5f9e-a71f-4a5b-8595-2a80174ea58f","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.16103044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"000d5f9e-a71f-4a5b-8595-2a80174ea58f","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:10.268334144Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:10.631834305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"000d5f9e-a71f-4a5b-8595-2a80174ea58f","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":505664,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.667303219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"000d5f9e-a71f-4a5b-8595-2a80174ea58f\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:10 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 3.52935939s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:10.668336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"000d5f9e-a71f-4a5b-8595-2a80174ea58f\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"000d5f9e-a71f-4a5b-8595-2a80174ea58f\" }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:10.865699109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.866629146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:10.874684334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:13.98420632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":34,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:13.985081919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:13.986430849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:13.987205677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:13.987918643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.079591543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.519020536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.521511713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.522686136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.523516876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.561220048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.568522451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.741770706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.742827475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.743900757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.750747314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.858790416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.959606525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.967987942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.978728533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:14.981311644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.985902752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:14.998599499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.024791007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"9899c3cc-2deb-448a-bceb-b6645b5496b1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.025798699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"9899c3cc-2deb-448a-bceb-b6645b5496b1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.02818604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.029909613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.033203916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.03382531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.034411114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.043706585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.053847025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"9899c3cc-2deb-448a-bceb-b6645b5496b1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.05480128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"9899c3cc-2deb-448a-bceb-b6645b5496b1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.069051081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"9899c3cc-2deb-448a-bceb-b6645b5496b1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.081642238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.082618748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":34,"elapsed":"1.097902554s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.083787479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.262237105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.26316113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.263761933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.268592798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.303777968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.305346073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.30646259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.307443178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.317772072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.356431683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"fb0be275-b73a-4fef-9763-7158d8d0855e","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.358683265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"fb0be275-b73a-4fef-9763-7158d8d0855e","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:15.638455676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.744988008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.859059253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:15.86338268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"9899c3cc-2deb-448a-bceb-b6645b5496b1","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":839018,"response_code":200}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:15.940647945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.94183364Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:15.969553504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:16.034872903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"fb0be275-b73a-4fef-9763-7158d8d0855e","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":678232,"response_code":500}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:16.124901758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:16.125845212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:16.126725079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:16.127423907Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:16.133261211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:16.190879526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb0be275-b73a-4fef-9763-7158d8d0855e\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:16 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 3.216202888s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:16.192111552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb0be275-b73a-4fef-9763-7158d8d0855e\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb0be275-b73a-4fef-9763-7158d8d0855e\" }"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:16.528753171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:16.689413074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:17.392084527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:17.394131032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:17.395103283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:17.398301535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":13,"elapsed":"2.365161446s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:17.970749223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:17.973170843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:17.974177883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:17.974919345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.01819796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.051173803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:18.052006066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.052775875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":5,"elapsed":"493.55µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.05347318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.054096217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.054735577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.055353404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.055939488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.056955406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.062205192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.066848133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.067629565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.068707637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:18.069432316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.070045554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":5,"elapsed":"375.616µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.070719843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:18.071307781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 07:00:18.068954667 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.071886941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.072573194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.087979973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.472591113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.497967793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.507001484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.563825462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.68696648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.708511259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.717135298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.718630217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.71943253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.740606313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.78991827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.792334138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.821888195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.839275788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.840197419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.847329363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.883753064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:18.884829522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.885669931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":5,"elapsed":"475.044µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.88660094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.887470116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.888576694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.892520006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.899672882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.900671776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.901644678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.906732383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.916156851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:18.91772741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.919209323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:18.966449897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.083693983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.111619636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.112502599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.11973324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.171945869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.172751138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.173333875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.17391003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.174451265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.175286293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.17588371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.176621385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.177385502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.178112025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.182321796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.189153214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:19.190009895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.190568114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.191085111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:19.191684682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.192215386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.19291259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.193634153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.198553634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.206589814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.337501075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":35,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.338972097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.34029315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.341648362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.342777044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.35247757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.377563472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:20323","req_id":"977064ff-86a0-43ae-845a-a72bd2802881","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   d9 45 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   .E..........._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.378637636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:20323","req_id":"977064ff-86a0-43ae-845a-a72bd2802881","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55621,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55621,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:19.393573149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:20323","req_id":"977064ff-86a0-43ae-845a-a72bd2802881","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:19.400057036Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3000","req_id":"d111a9bc-84e9-473c-ac1f-8b4fc19920bc","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   cc ca 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   ............._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.401713817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3000","req_id":"d111a9bc-84e9-473c-ac1f-8b4fc19920bc","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 52426,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 52426,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:19.443013421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3000","req_id":"d111a9bc-84e9-473c-ac1f-8b4fc19920bc","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.501818653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.796242753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.831600336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":5,"elapsed":"931.601697ms"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:19.864354459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 07:00:18.068954667 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.865445586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T07:00:18.068954667Z\",\"pruned\":[]}","iteration":5,"elapsed":"1.795356927s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:19.890767347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:20.06437713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:20.065656141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:20.200432067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:20.201398126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:20.202119649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:20.253461448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.34109391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:23.387722879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:35602"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:23.403626164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.404627833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":5,"elapsed":"5.351954102s"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:23.410584794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.411905948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":5,"elapsed":"5.344649644s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.498384842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.49938553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.504631058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.506421975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.507335699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.5082425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.508952631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.515747596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:23.581997424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.583127649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":5,"elapsed":"5.528030263s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.583990542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.584920329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.585965264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.587360906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.588083892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.592162122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.601640877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.602615513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.603328088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.635810922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.642591669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.643368312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.644091267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.651485954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.655969128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.662787369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.66368392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":9,"elapsed":"724.759µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.664378069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:23.664992679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.665539255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":9,"elapsed":"159.116µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.666190067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.66690623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.667619336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.668656265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.669384782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.67842132Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.687332238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.68817444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.688885773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.689629881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.690312927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.696614292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.709375387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:23.710163202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.710730659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":9,"elapsed":"1.302867ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.711543393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.71214127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.713314731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62907","req_id":"e14f4a2d-0699-4971-aab9-ba56743561cf","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   2c d9 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ,............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.714290138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62907","req_id":"e14f4a2d-0699-4971-aab9-ba56743561cf","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 11481,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 11481,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.721691117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62907","req_id":"e14f4a2d-0699-4971-aab9-ba56743561cf","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.722279055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42103","req_id":"144bb53f-ffbc-4233-a6d9-de2bf1aa0161","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   ab 35 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .5..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.723199783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42103","req_id":"144bb53f-ffbc-4233-a6d9-de2bf1aa0161","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43829,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43829,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.728452516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42103","req_id":"144bb53f-ffbc-4233-a6d9-de2bf1aa0161","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.728992949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54564","req_id":"a7e8a593-8500-4ee0-893b-3f70a0489939","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   2d d5 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   -............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.729887836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54564","req_id":"a7e8a593-8500-4ee0-893b-3f70a0489939","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 11733,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 11733,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.751900606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54564","req_id":"a7e8a593-8500-4ee0-893b-3f70a0489939","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.782200584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64287","req_id":"a29083a4-bb90-4ed8-bfbd-2cab661d7dfa","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   4b b7 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   K............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.783360738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64287","req_id":"a29083a4-bb90-4ed8-bfbd-2cab661d7dfa","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 19383,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 19383,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.828258839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64287","req_id":"a29083a4-bb90-4ed8-bfbd-2cab661d7dfa","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.828887608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"d0c3c654-3173-4638-be63-3f09c6cf3888","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.829611486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"d0c3c654-3173-4638-be63-3f09c6cf3888","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.830321315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.831278717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.832091752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.832830268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":5,"elapsed":"4.605326297s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.833700667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.917038192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.918041956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.918803328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.920260021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.958759745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:23.959845541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:23.960642324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.961347795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:23.980975547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:24.085742764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:24.086607281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.087205679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":5,"elapsed":"4.985582978s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.087869327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"7c2e1f68-aa40-454e-a043-9548b8004b8f","remote_addr":"[::1]:35602","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.088575239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"7c2e1f68-aa40-454e-a043-9548b8004b8f","remote_addr":"[::1]:35602","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.089235741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5416","req_id":"f2570f81-75e3-48a9-8d93-8eec124440a0","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   00 15 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.137875243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5416","req_id":"f2570f81-75e3-48a9-8d93-8eec124440a0","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 21,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 21,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:24.146455264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5416","req_id":"f2570f81-75e3-48a9-8d93-8eec124440a0","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.147055195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52135","req_id":"0c707c86-bd4c-45ab-b0fc-d9c5d218defd","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   8d 4a 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .J..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.148240098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52135","req_id":"0c707c86-bd4c-45ab-b0fc-d9c5d218defd","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 36170,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 36170,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:24.156983724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52135","req_id":"0c707c86-bd4c-45ab-b0fc-d9c5d218defd","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:24.157600289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"7c2e1f68-aa40-454e-a043-9548b8004b8f","remote_addr":"[::1]:35602","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":71264,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.160197958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.301671867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.303165563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.305398949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.307323762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.309347531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.327974145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:24.443513005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.449399646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":5,"elapsed":"6.479510371s"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.469297014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:24.51246048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.514448769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":5,"elapsed":"5.630111345s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.52285248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.819816113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:24.820974223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.821710365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.826794833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:24.884324242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:24.885432783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":9,"elapsed":"1.174262722s"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.036893657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c2e1f68-aa40-454e-a043-9548b8004b8f\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:10:24 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:25.038072218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:25.038927757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:25.039542628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.040652082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":7,"elapsed":"15.013986432s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:25.201506722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:25.202503923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:25.203268842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.2039729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.210148145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:25.361595823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:25.36261181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.363342031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.369488157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.416285942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.671156616Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:25.672397701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.673135476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.680999986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:25.756142415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.011191463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.012320365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.013579135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.018326103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.077993487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.083242171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.088104208Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.191305735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.192353866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.19287446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.214936537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.253720726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":9,"elapsed":"2.597474318s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.677563543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.896040226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.896881186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.897391389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.903643596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.952770146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21182","req_id":"89017544-e9e3-4c75-b66a-b4322e77d2fe","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   cb 82 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.954229534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21182","req_id":"89017544-e9e3-4c75-b66a-b4322e77d2fe","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 52098,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 52098,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:26.958810022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:21182","req_id":"89017544-e9e3-4c75-b66a-b4322e77d2fe","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:26.95924201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:48571","req_id":"1bbf377d-725b-4f8c-b8ea-1f31c539b1aa","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   c5 6c 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .l..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.959888894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:48571","req_id":"1bbf377d-725b-4f8c-b8ea-1f31c539b1aa","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50540,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50540,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:26.963694563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:48571","req_id":"1bbf377d-725b-4f8c-b8ea-1f31c539b1aa","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:26.99597308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:26.996707269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.997516346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":9,"elapsed":"3.395028997s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:26.998166186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:26.998668484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:26.999163347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":9,"elapsed":"3.356745182s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.014666788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.332639017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:27.34271079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.343459297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.363679015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.419148312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.626906186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:27.627768468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.628288942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.636027502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:27.646560411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"d0c3c654-3173-4638-be63-3f09c6cf3888","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":3872238,"response_code":500}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.648296348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.648997511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":35,"elapsed":"8.310961113s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.649476041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":36,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:27.649901466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:27.650528201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:27.651055268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.651549449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.664115376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:27.696106263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.696752587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":5,"elapsed":"8.780321702s"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.762132897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0c3c654-3173-4638-be63-3f09c6cf3888\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:27 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 5.588052701s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:27.822581272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0c3c654-3173-4638-be63-3f09c6cf3888\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0c3c654-3173-4638-be63-3f09c6cf3888\" }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.912213479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":16,"elapsed":"4.327581954s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:27.977198715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":16,"elapsed":"4.472374173s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.132643511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":9,"elapsed":"4.468206355s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.134222878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.325197309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:28.333103331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.334111062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.341771026Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.644186528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:28.645247816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:28.645828649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:28.646845579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:28.647955403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.648667327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.670527954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.949650394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.950684768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":36,"elapsed":"1.301064771s"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:28.973433509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:28.974321763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":5,"elapsed":"9.799697075s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:29.138855882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:29.597059071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:29.606928232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:29.608499402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:29.609473667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:29.610458283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:29.616946008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.139462839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.244753485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:30.246160209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.24690075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.250336574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.281536229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:30.282576855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.283498065Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.296837628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:30.400749386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:30.402066221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:30.40310808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.404292553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:30.430228374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:33.836235785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.837571478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:33.892114184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.893631938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":5,"elapsed":"15.102641118s"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:33.894692925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.895455981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":9,"elapsed":"10.207570975s"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.896228285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.896793467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":5,"elapsed":"14.70698371s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:33.897567445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:33.898273076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:33.898996242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.899683637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.935613307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:33.946365281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.025564577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.102942396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.103942342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.110996322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.245015565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.248432622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.249228853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.249934545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.251982081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.259174035Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.346880517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.347985743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.348706354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.368783188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:34.394498952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:36807"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.429646197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":37,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.440390185Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.442935791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.44531168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.446340193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.449946545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.45798482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.459539227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.510563766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.700258686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.702567782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.911484697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.912349335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.912873946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.927655263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:34.94604169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"ea4317bb-0517-4f13-9dcd-bae4b1f86fc6","remote_addr":"[::1]:36807","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:34.946879875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"ea4317bb-0517-4f13-9dcd-bae4b1f86fc6","remote_addr":"[::1]:36807","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:35.117136278Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f63dcf07-aaf4-4a87-b141-8ddc5f7d1b86","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:35.118032468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f63dcf07-aaf4-4a87-b141-8ddc5f7d1b86","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:35.150611323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:35.159246332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"ea4317bb-0517-4f13-9dcd-bae4b1f86fc6","remote_addr":"[::1]:36807","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":213170,"response_code":200}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.533352485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:38.581670719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:57962"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.620643683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.621800671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.626804068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.628289368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.629367259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.642617515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.650099224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52380","req_id":"0cd809be-7e68-4350-a472-0eecee25606c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   63 bb 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   c............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.652324904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52380","req_id":"0cd809be-7e68-4350-a472-0eecee25606c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 25531,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 25531,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:38.658019965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52380","req_id":"0cd809be-7e68-4350-a472-0eecee25606c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.658571962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51356","req_id":"1b303eb4-38e3-4cbe-ad1c-ab24aebfdbba","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   87 fe 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.659478131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51356","req_id":"1b303eb4-38e3-4cbe-ad1c-ab24aebfdbba","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 34814,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 34814,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:38.664654903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51356","req_id":"1b303eb4-38e3-4cbe-ad1c-ab24aebfdbba","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.66516706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea4317bb-0517-4f13-9dcd-bae4b1f86fc6\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:10:35 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.666151996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.666845453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.671296083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.719194413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.720186163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":37,"elapsed":"4.28984211s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.76498653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"a3e40c43-73c7-4f3c-912b-9c1ea5543eaa","remote_addr":"[::1]:57962","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.766260089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"a3e40c43-73c7-4f3c-912b-9c1ea5543eaa","remote_addr":"[::1]:57962","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.767222591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.767826721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.787003807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.787925758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.788647722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.796839472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:38.805423841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58839","req_id":"b98aa4a9-7289-48e3-88ca-da141dc49699","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   4a 71 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   Jq..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:38.80856025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58839","req_id":"b98aa4a9-7289-48e3-88ca-da141dc49699","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 19057,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 19057,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:38.832903287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58839","req_id":"b98aa4a9-7289-48e3-88ca-da141dc49699","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:38.94663585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"a3e40c43-73c7-4f3c-912b-9c1ea5543eaa","remote_addr":"[::1]:57962","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":136159,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.086502797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"a3e40c43-73c7-4f3c-912b-9c1ea5543eaa\", \"date\": \"Wed, 04 Mar 2026 07:10:38 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.107739124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:39.108904198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.109768845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":5,"elapsed":"19.875560559s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.1825122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.380780006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:39.381846695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.382570492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.400253602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.454620279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":38,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:39.455788419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:39.456800259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:39.457889853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.459259533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.474813114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.523503156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.72793276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:39.728873959Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.729589831Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.734831732Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.752549611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":17,"elapsed":"1.131912161s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:39.758298289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.075907145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:40.076998934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.077849633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.122691343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.228866165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:40.230020076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.231020523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.239166682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:40.262298714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:40.263252799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:40.264152436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.264702809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":14,"elapsed":"9.981945215s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.268718545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.269687179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":38,"elapsed":"816.023689ms"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:40.27615298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":17,"elapsed":"1.509799104s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:41.213043973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:41.21402923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:41.222198945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:42.494795226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.507756679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:42.520161506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"150bd6a3-4d99-4c3b-9088-c237d7844b82","remote_addr":"[::1]:36807","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.521759389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"150bd6a3-4d99-4c3b-9088-c237d7844b82","remote_addr":"[::1]:36807","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:42.529009849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"150bd6a3-4d99-4c3b-9088-c237d7844b82","remote_addr":"[::1]:36807","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":8673,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.578096119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"150bd6a3-4d99-4c3b-9088-c237d7844b82\", \"date\": \"Wed, 04 Mar 2026 07:10:42 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.599611921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":10,"elapsed":"13.954894369s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.600456097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:42.601085327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:42.60163038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:42.602341422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:42.603076241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.603769268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.612412945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:42.805734457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.095415908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:43.096437067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:43.097176194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:43.098456998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.099184343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.104696811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.26083286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.554222456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:43.55553328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.556271235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:43.562358104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:46.540798662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.541993485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.546912274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.547790178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.548535999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.549227663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.602677829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:46.671716225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f63dcf07-aaf4-4a87-b141-8ddc5f7d1b86","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":11554418,"response_code":500}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:46.678269731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:58757"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.684821844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"985560ac-7a57-4856-9a21-55a62090e627","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.685458709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"985560ac-7a57-4856-9a21-55a62090e627","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.685945626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":39,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.68652142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.687047605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.687579361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.688075757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.693623335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.704893208Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.706329921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.707237483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.745326201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.751749197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f63dcf07-aaf4-4a87-b141-8ddc5f7d1b86\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:46 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 5.820407615s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:46.752748252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f63dcf07-aaf4-4a87-b141-8ddc5f7d1b86\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:10:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f63dcf07-aaf4-4a87-b141-8ddc5f7d1b86\" }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.782288941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"985560ac-7a57-4856-9a21-55a62090e627","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.783408114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"985560ac-7a57-4856-9a21-55a62090e627","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.789921049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"985560ac-7a57-4856-9a21-55a62090e627","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.831990473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.973068806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.974067119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.975098026Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:46.975959437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.976653956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:46.980763578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.09510491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.096180215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.097061035Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.09776851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.109380011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.121350095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.26132674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.262612163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.263353736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.28038956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.288100375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"26d841f5-8179-4e27-93a5-e446f2c584f4","remote_addr":"[::1]:58757","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.288986084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"26d841f5-8179-4e27-93a5-e446f2c584f4","remote_addr":"[::1]:58757","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:47.296701467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"985560ac-7a57-4856-9a21-55a62090e627","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":611809,"response_code":200}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:47.301242316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"26d841f5-8179-4e27-93a5-e446f2c584f4","remote_addr":"[::1]:58757","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":12743,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.30403397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.498898778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.499877421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.500896475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.503908548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.56666634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.628369658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.63277638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.633978137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":39,"elapsed":"946.822929ms"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.647581163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26d841f5-8179-4e27-93a5-e446f2c584f4\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:10:47 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.649143326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.649907604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.657148284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:47.923480281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.924426982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:47.928308701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.083958915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.08496317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.085722728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.086424001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.111292001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.274000674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.407816934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:48.470941705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.472569889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.484087985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"3a358a93-4cde-4074-9974-19f3b5163c26","remote_addr":"[::1]:58757","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.485019234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"3a358a93-4cde-4074-9974-19f3b5163c26","remote_addr":"[::1]:58757","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:48.547420418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"3a358a93-4cde-4074-9974-19f3b5163c26","remote_addr":"[::1]:58757","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":63486,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.55845806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"3a358a93-4cde-4074-9974-19f3b5163c26\", \"date\": \"Wed, 04 Mar 2026 07:10:48 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.562107598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":11,"elapsed":"5.96215926s"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:48.83433329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.835149171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:48.835848991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.836406939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":15,"elapsed":"2.130806367s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.906461242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64061","req_id":"a894c2a8-d546-4f74-9013-6a4b830017f8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   54 1f 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   T............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.910862895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64061","req_id":"a894c2a8-d546-4f74-9013-6a4b830017f8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 21535,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 21535,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:48.916402917Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64061","req_id":"a894c2a8-d546-4f74-9013-6a4b830017f8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:48.916963932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64643","req_id":"c1344c06-ed0c-44c4-98e6-ae35dfe35d49","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   47 2b 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   G+..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:48.917887706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64643","req_id":"c1344c06-ed0c-44c4-98e6-ae35dfe35d49","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 18219,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 18219,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:48.930264327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64643","req_id":"c1344c06-ed0c-44c4-98e6-ae35dfe35d49","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:49.279816727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57105","req_id":"1a0c67d2-3192-493a-81b5-fd0a8a31c868","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   89 3e 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .>..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:49.280959858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57105","req_id":"1a0c67d2-3192-493a-81b5-fd0a8a31c868","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 35134,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 35134,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:49.287119241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57105","req_id":"1a0c67d2-3192-493a-81b5-fd0a8a31c868","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.408051136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:55.501729599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:65295"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.504264032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":40,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.505456621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.5133009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.514324934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.515093621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.536862325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.573749326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.57847426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.581631841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.582638741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.584171836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.607167397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.692783437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.730836604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.763434466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.764637576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.765474989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.919597489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.925362629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:55.926260592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.926971214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:55.933827651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.060820329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.069283347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.070061703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":10,"elapsed":"380.185µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.070738016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:10:56.071347817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.071892048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":10,"elapsed":"192.132µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.072553472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.073138784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.073842803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.106569851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.184110684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.263705416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.478994978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.499933304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.502370334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.530661903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.5384193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.571479771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.632976188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:56.634045542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.635129525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":10,"elapsed":"739.699µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.636277495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.636887225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.637400154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.637975256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.639219768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.640199083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.64089729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.650797082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.715795975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"e31c82d5-d986-40b4-bb39-3f262967b41b","remote_addr":"[::1]:65295","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.716828475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"e31c82d5-d986-40b4-bb39-3f262967b41b","remote_addr":"[::1]:65295","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.770064467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"36a3245d-cdfc-44f0-8711-e04184cb2261","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.771070354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"36a3245d-cdfc-44f0-8711-e04184cb2261","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:56.860326524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:56.86817409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"e31c82d5-d986-40b4-bb39-3f262967b41b","remote_addr":"[::1]:65295","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":152592,"response_code":200}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:56.951041472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:12275","req_id":"7d2a08ea-0082-4bb3-9c8e-d3b562604ab2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   9c dc 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.952145935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:12275","req_id":"7d2a08ea-0082-4bb3-9c8e-d3b562604ab2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 40156,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 40156,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:56.957850705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:12275","req_id":"7d2a08ea-0082-4bb3-9c8e-d3b562604ab2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.958406169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e31c82d5-d986-40b4-bb39-3f262967b41b\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:10:56 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:56.959295665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:56.959892941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:56.960476631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.961058146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":8,"elapsed":"16.728926005s"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:56.965213248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:56.98439261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":10,"elapsed":"350.434752ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.183591355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.450692159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:57.451466577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.451931902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.483512904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.586130922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.833057448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:57.847620251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.864943106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.888476655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:57.897881624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:57.919991837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:57.985381506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.149539678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.15047225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.151082142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.159098673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.204430326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.395724211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.396799467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.397528124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.404159625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.42499226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":10,"elapsed":"2.363970581s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.505712537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.507113008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":40,"elapsed":"3.001102537s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.545930894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59078","req_id":"6ae6fdbd-1fa4-4994-b7ab-405be3de59b3","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   69 65 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ie..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.553379648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59078","req_id":"6ae6fdbd-1fa4-4994-b7ab-405be3de59b3","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26981,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26981,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:58.566995059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59078","req_id":"6ae6fdbd-1fa4-4994-b7ab-405be3de59b3","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.567520513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10219","req_id":"3f1ce230-b7d4-4e0d-b6e9-28f3704f3e80","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   20 05 01 00  00 01 00 00  00 00 00 01  09 5f 64 65    ............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.568109162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10219","req_id":"3f1ce230-b7d4-4e0d-b6e9-28f3704f3e80","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 8197,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 8197,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:58.57183411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:10219","req_id":"3f1ce230-b7d4-4e0d-b6e9-28f3704f3e80","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.739328808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":18,"elapsed":"3.046218652s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.740369014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":10,"elapsed":"2.670398733s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:58.759537715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:58.760527571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.761243653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":10,"elapsed":"3.17906683s"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:58.817231179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:58.818146426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.818851336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":10,"elapsed":"2.892692336s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.823139002Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:10:58.823861898Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.824403634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.825108904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:58.825828444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.826637661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:58.909384212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:59.066663582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52702","req_id":"7670b702-c494-416d-ad87-319e1bcdca52","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   f3 34 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .4..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:59.06800177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52702","req_id":"7670b702-c494-416d-ad87-319e1bcdca52","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62260,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62260,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:59.137463466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52702","req_id":"7670b702-c494-416d-ad87-319e1bcdca52","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:59.138153727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64485","req_id":"35b4ad93-0f36-47db-9284-26f098e82f06","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   37 9e 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   7............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:59.139409862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64485","req_id":"35b4ad93-0f36-47db-9284-26f098e82f06","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 14238,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 14238,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:59.144716812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64485","req_id":"35b4ad93-0f36-47db-9284-26f098e82f06","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:10:59.443375022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64960","req_id":"26300ff8-da93-4e76-a064-1cfad34c5a8d","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   0d f3 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:59.455760591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64960","req_id":"26300ff8-da93-4e76-a064-1cfad34c5a8d","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 3571,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 3571,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:10:59.465063137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64960","req_id":"26300ff8-da93-4e76-a064-1cfad34c5a8d","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:10:59.828049537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.194986193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.27088239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.272521766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.273760115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.274686084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.299215502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.532762014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":41,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.534393865Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.535337289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.53626928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.53698944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.567586679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.646625076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.901048171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:00.90200954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.902735392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:00.909056406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:01.15140459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:01.319067292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:01.320208388Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:01.320950392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:01.326322251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:03.912693807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:03.913610728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:03.919122275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:03.976645451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:03.977666448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:03.978394164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:03.981180618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:04.001064739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.004789637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":10,"elapsed":"7.525727555s"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.01225314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.014249856Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":41,"elapsed":"3.482115068s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.021305048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":18,"elapsed":"7.385541184s"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:04.05976361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:46876"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.088476506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.089270734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.089994842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.111678307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.134505544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"3f5896b7-e357-4931-b76f-e1e03e28cb41","remote_addr":"[::1]:46876","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.136891683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"3f5896b7-e357-4931-b76f-e1e03e28cb41","remote_addr":"[::1]:46876","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:04.217552171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"3f5896b7-e357-4931-b76f-e1e03e28cb41","remote_addr":"[::1]:46876","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":83294,"response_code":200}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.224096088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.2255369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.227267086Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.228490355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.238178226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.258628613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f5896b7-e357-4931-b76f-e1e03e28cb41\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:11:04 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.260924953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.262752733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.27315342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.380294912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.429479517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.586145679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.587121537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.612652684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.71704251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.717977598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:04.719056591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.719792051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:04.751651056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.006893007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.109806983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:05.112324783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.113377734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:05.223694152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"35e2e28d-097a-4fc7-b142-dc9be7113ebc","remote_addr":"[::1]:46876","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.227138984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"35e2e28d-097a-4fc7-b142-dc9be7113ebc","remote_addr":"[::1]:46876","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:05.243047369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"35e2e28d-097a-4fc7-b142-dc9be7113ebc","remote_addr":"[::1]:46876","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":19390,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.284903148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"35e2e28d-097a-4fc7-b142-dc9be7113ebc\", \"date\": \"Wed, 04 Mar 2026 07:11:05 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.29968675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":12,"elapsed":"6.47664475s"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:05.609736083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:05.612790109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:05.613904973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.614898286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":16,"elapsed":"1.521847083s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.617872994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":42,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:05.618997768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:05.619912825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:05.621248728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.621973598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.649930663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:05.856612851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:06.13417414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:06.134922386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:06.135470505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:06.158822905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:06.20299259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:06.204341448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":42,"elapsed":"585.273061ms"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:06.300275842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"36a3245d-cdfc-44f0-8711-e04184cb2261","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":9530170,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:06.471238147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36a3245d-cdfc-44f0-8711-e04184cb2261\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:06 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 10.444460745s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:06.504008753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36a3245d-cdfc-44f0-8711-e04184cb2261\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"36a3245d-cdfc-44f0-8711-e04184cb2261\" }"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:09.550818337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59956","req_id":"e8c5ce90-a743-4e06-bb6b-d7bc328c5f0b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   c3 5e 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .^..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:09.562696137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59956","req_id":"e8c5ce90-a743-4e06-bb6b-d7bc328c5f0b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50014,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50014,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:09.587378619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59956","req_id":"e8c5ce90-a743-4e06-bb6b-d7bc328c5f0b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:09.588562721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56415","req_id":"f5562d58-90e0-4742-8802-1d3c17588963","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   f5 8a 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:09.589570352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56415","req_id":"f5562d58-90e0-4742-8802-1d3c17588963","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62858,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62858,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:09.611437532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56415","req_id":"f5562d58-90e0-4742-8802-1d3c17588963","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:09.633866608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59871","req_id":"4a77c448-0aa7-4419-a025-6200ee90673c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   3d 7e 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   =~..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:09.635178123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59871","req_id":"4a77c448-0aa7-4419-a025-6200ee90673c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 15742,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 15742,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:09.640051642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59871","req_id":"4a77c448-0aa7-4419-a025-6200ee90673c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.239260419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:10.240194494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.241050715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.249042607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.721125784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":43,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:10.721946003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:10.722705481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:10.723452675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.724280158Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.752072707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:10.895924408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.897100333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:10.95045977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.001401234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.18227307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.183255261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.183693852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.190925264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.297347327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.297936237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.298371511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.29879393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.299187832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.306327462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.550816708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.551589453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:11.55203611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.552435102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.562672916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.605413903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.627983007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":43,"elapsed":"906.607838ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:11.668883108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.19254867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.193762571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.194499194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.200898935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.210560283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.389814767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.425837742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.491884326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.511272713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.520381523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.548945674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.623785342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.626765912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.628206843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:12.629008646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.629708466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.634196209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.738208957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":19,"elapsed":"1.440738445s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:12.74054705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.034989306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:13.049943176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.080583751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.119638887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.146234098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.297998965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:13.299442602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.300333332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:13.302416439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.565667712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:15.60366107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:34071"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.623804326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"e59d7d24-fa18-495f-8078-30592ec85b11","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.624740746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"e59d7d24-fa18-495f-8078-30592ec85b11","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.629781718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"starting background task for service firewall rule propagation","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.631919584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.633115239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.642885203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.653670884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.663132806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.664007032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.664743194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.665484046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.666174998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.67292316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.679712074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":4,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.68070726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.68143202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.682765037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.684112423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.715764357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.868148744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"e59d7d24-fa18-495f-8078-30592ec85b11","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.869381292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"e59d7d24-fa18-495f-8078-30592ec85b11","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.877378415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"e59d7d24-fa18-495f-8078-30592ec85b11","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.901364814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":19,"elapsed":"3.277724059s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.922213902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"3d1adca2-3f74-4b53-933e-8c37ca455ed3","remote_addr":"[::1]:34071","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.923097146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"3d1adca2-3f74-4b53-933e-8c37ca455ed3","remote_addr":"[::1]:34071","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.936816043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":44,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.937633667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.938504206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:15.93926653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.939979656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.950383329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:15.973276637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"3d1adca2-3f74-4b53-933e-8c37ca455ed3","remote_addr":"[::1]:34071","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":51106,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:15.976585649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.125060693Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.126669628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.12740564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.133029359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.139889023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.140724112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.146812263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.154049677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.2449937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.24592052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.246523197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.254354661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:16.263087365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"e59d7d24-fa18-495f-8078-30592ec85b11","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":639634,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.281819449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.404608658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.408637259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.4096951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.458098443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read target blueprint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:16.472440017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: no target blueprint set"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.47524125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read target blueprint: Internal Error: no target blueprint set\"}","iteration":4,"elapsed":"794.74067ms"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.50061282Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d1adca2-3f74-4b53-933e-8c37ca455ed3\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:11:15 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:16.501668517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:16.502289641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:16.502885664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.503472219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":9,"elapsed":"6.263150582s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.547728015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.548423797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.548867218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.549261922Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.555238271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.575923263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.576967617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.583257129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.611530582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.613372871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":44,"elapsed":"677.074787ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.693398118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.839444377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.840524402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.841251116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.871106341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.880887006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.883573612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.884848524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.886665092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.896460967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.965662337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.967379046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.968885358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.977741928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.988370938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"TargetReleaseConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.98986253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:16.990978346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.991959094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:16.996353041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.015125705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.018209981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.061973639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"7f4e5bf6-9eb5-43b4-929b-460de35c621f","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.062965429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"7f4e5bf6-9eb5-43b4-929b-460de35c621f","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:17.065143114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.100545446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.101623407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.102368206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.120263086Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.150410893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.250883119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"TargetReleaseConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.252290263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.253304177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.257702783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:17.263347915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"7f4e5bf6-9eb5-43b4-929b-460de35c621f","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":201675,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.28509692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f4e5bf6-9eb5-43b4-929b-460de35c621f\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:17 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 16.695354504s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:17.285782171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f4e5bf6-9eb5-43b4-929b-460de35c621f\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7f4e5bf6-9eb5-43b4-929b-460de35c621f\" }"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.286924059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.389728718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Vpc\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.391059982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.391633621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.39266037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.407068115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.407870799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.408461983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"tuf_repo_prune decision","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:17.613633321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"TufRepoPrunerDetails { nkeep_recent_releases: 3, nkeep_recent_uploads: 2, repos_keep_target_release: {}, repos_keep_recent_uploads: {}, repo_prune: None, other_repos_eligible_to_prune: {}, warnings: [] }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.614388671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"tuf_repo_pruner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"nkeep_recent_releases\":3,\"nkeep_recent_uploads\":2,\"other_repos_eligible_to_prune\":[],\"repo_prune\":null,\"repos_keep_recent_uploads\":[],\"repos_keep_target_release\":[],\"status\":\"enabled\",\"warnings\":[]}","iteration":3,"elapsed":"1.925759637s"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.663025989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.66394835Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.664775453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.696322899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.819542834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.84044655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.846348211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.862683775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.98006788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.980667891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:17.98103489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.981446648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:17.992407166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.046524648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:18.096853686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.098276091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":6,"elapsed":"50.10874ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.128727861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.129685503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.130467927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.131189029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.136933708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.170782845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.17159584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.173160758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.173890978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.174492062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.175530464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.181626021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.213405037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.214220546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.215971694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:18.216676895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.217609968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":6,"elapsed":"435.284µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.218299988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:18.218892004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 07:01:18.216378652 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.219675139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.22037041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.224013576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.248669635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.382728657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.383764915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.384291161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.386313978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.394143839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.553278604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.556349833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.557518384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.558316449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.559148692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.563264486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:18.670494223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 07:01:18.216378652 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.671411355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T07:01:18.216378652Z\",\"pruned\":[]}","iteration":6,"elapsed":"454.670263ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.674161517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.674879443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.683538561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.73908966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.964600793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:18.981900522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.982792824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:18.989466759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:19.028752263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.031242289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":6,"elapsed":"857.810262ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.070962296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.072437085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.073287404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.073912816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.074486826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.079647185Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.086556417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:19.08716198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.087617244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":6,"elapsed":"772.434µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.088207887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.088690135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.089259716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.094628369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.0998252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.100404751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.100981406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.115308551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.14434516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.145846552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.146891237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.153516886Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.15963649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61579","req_id":"fdac3669-0d6b-4de6-85dd-d692c4f6ddfe","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   51 23 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   Q#..........._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.162343156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61579","req_id":"fdac3669-0d6b-4de6-85dd-d692c4f6ddfe","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 20771,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 20771,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:19.170064701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61579","req_id":"fdac3669-0d6b-4de6-85dd-d692c4f6ddfe","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.171973713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61053","req_id":"2ee06020-6ad2-463e-917c-79d4a0f2a720","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   47 7f 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   G............_ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.173906953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61053","req_id":"2ee06020-6ad2-463e-917c-79d4a0f2a720","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 18303,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 18303,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:19.181294355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61053","req_id":"2ee06020-6ad2-463e-917c-79d4a0f2a720","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.195977807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.197412496Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.211377392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:19.255276499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.260317742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":6,"elapsed":"1.088166018s"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:19.263119105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.264024313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":6,"elapsed":"1.050572303s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.270414244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.271194413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.271924373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.283213274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.289327638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.290272314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.291342539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.311972763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.389262647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.390043307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.390525625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.391000418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.391445332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.392438705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.393004939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.393619871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.394379599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.395267884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.401612284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.409270855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.510067716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.545596038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.546626514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.547450701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.548204238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:19.54882436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.54937346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.55008332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:19.550928879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.551467629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.552163401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.552883241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.553598842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.561892384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.654733313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":6,"elapsed":"554.806572ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.656823284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.65800921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.658787135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.659504219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.679225066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.736161066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.737792536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.739020234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.740616034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.75100141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.855354122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:19.870511176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.872472842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:19.922900976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:19.994746648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.00396692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":6,"elapsed":"729.779723ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.066601908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.068324369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.069439824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.070342688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.074647768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.147672741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.148496678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.149226748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.155637701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.231286535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3038","req_id":"fe8240d1-ab1c-4191-b790-2ebe17597b2e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   74 e0 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   t............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.232306571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3038","req_id":"fe8240d1-ab1c-4191-b790-2ebe17597b2e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 29920,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 29920,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.236578035Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3038","req_id":"fe8240d1-ab1c-4191-b790-2ebe17597b2e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.237262895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:39603","req_id":"dab6f354-5512-4034-b43d-031bcfccbf9a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   7d 8d 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   }............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.238032473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:39603","req_id":"dab6f354-5512-4034-b43d-031bcfccbf9a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32141,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32141,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.242266783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:39603","req_id":"dab6f354-5512-4034-b43d-031bcfccbf9a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.29767097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.298571609Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.299331989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.300072831Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":6,"elapsed":"897.956696ms"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.306913166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.307733445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.308539847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.309243715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.310106689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.387832268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50934","req_id":"08858048-d8d0-4e5b-b5c9-d9940e63897d","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   49 ed 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   I............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.392320161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50934","req_id":"08858048-d8d0-4e5b-b5c9-d9940e63897d","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 18925,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 18925,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.398917084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50934","req_id":"08858048-d8d0-4e5b-b5c9-d9940e63897d","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.399832381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62669","req_id":"bb45ffcc-414d-46d7-9c58-1e47032e12a5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   82 2c 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .,..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.400796166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62669","req_id":"bb45ffcc-414d-46d7-9c58-1e47032e12a5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33324,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33324,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.42889884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62669","req_id":"bb45ffcc-414d-46d7-9c58-1e47032e12a5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.469396882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.470552708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.471755988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.472746595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.47946582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.651933635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.653319307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.65407103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.661396199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:20.670001399Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.673045796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":6,"elapsed":"2.693068226s"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.677947671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.680188762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":6,"elapsed":"1.53587775s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:20.823526943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63630","req_id":"e85c436e-a9a3-4be7-baee-9aa0262871ab","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   80 35 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .5..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.831862528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63630","req_id":"e85c436e-a9a3-4be7-baee-9aa0262871ab","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32821,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32821,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:20.838682523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63630","req_id":"e85c436e-a9a3-4be7-baee-9aa0262871ab","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:20.83978849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.229847068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"Vpc\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:21.231159866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:21.231743555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.232456501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.238046573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:21.2680337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.269107272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":6,"elapsed":"2.197717728s"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.271538401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:21.272763013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.273751246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":6,"elapsed":"2.184542962s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.330205078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":45,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:21.330975478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:21.33171676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:21.332754391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.333478188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.341793173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:21.600099751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.601097072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.608208305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:21.846630361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.226718016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:22.228003499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.229002604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.266268918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:22.481426468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.485743583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.496868327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.658829696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.660371759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":45,"elapsed":"1.328325293s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.674300522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.78465262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:22.786056448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.791883601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:22.922639934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:22.986067286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.021843751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.029206765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.036727151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.19712786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.198056634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.198662889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.199380454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.199966017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.203722547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.345197939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.487217402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.489584623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.594202182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.59527863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.596002839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.601822517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.626185684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.627295609Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.627939768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.628533487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.632256481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.667219851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.668233655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.69604978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.815605868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.816459323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.817078463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.817764746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.820250302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.838802719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.839716514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.840322998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.889072509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.961838649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:23.96291164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.963777469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:23.969924438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.013868404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.136785948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.187611051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.424706202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"VpcSubnet\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.425834593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.426154077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.426722536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.427103653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.427458929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.429347891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.603446274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.60958687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.624884933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.668508964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.752570588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:24.753529402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.754264322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.771698527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:24.849318016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.197321269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName(\"external-dns\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.199091796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.20022177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.205612647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:25.213793635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.214704053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:25.215307662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.215848255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":17,"elapsed":"5.925294492s"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:25.216587694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.217349447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":6,"elapsed":"5.823553055s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.392077912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.393116695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.401266141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.553969893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.554991732Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.562678289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.597139091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.598099127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.604634227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.625243428Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.627682512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.629232821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.63768094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.677967833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.678524018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":6,"elapsed":"6.139591114s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.733978335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.734501884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.745024954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.75193109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.770067401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.787625171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.790692623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.792773676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.810069387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.837427723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:8239","req_id":"49cfb351-7403-4c38-b2c0-f30cb4b149e8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   11 a6 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.839798081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:8239","req_id":"49cfb351-7403-4c38-b2c0-f30cb4b149e8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 4518,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 4518,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:25.849132941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:8239","req_id":"49cfb351-7403-4c38-b2c0-f30cb4b149e8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.898805064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.900503166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.901688831Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.91240782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:25.923405131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:25.924796545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.926533915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":11,"elapsed":"303.128588ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.953014027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:25.953778696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.954491111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:25.96115729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.032961489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.034050612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.034914568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.063963432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.124506495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.159251021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.481990508Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"successful lookup of siloed resource \"VpcSubnet\" using built-in user","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.504998253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.507128504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.512602486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-4401-4000-8000-000000000000","resource_type":"Project","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.513602632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.51419045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.522257883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.541006699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.545847644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.546637553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":11,"elapsed":"607.878µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.547188737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:26.547521538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.54779919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":11,"elapsed":"116.933µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.549446702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.549824753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.550196731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.550571546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.550926761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.555307642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.594699197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.59548629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.596288043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.596828766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.597325703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.599874295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.608197265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":46,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.608796305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.609309113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.609830118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.610323067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.613666307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.657129709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.657761103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.658276196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.658797451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.659286302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.663928723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.682145372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3286","req_id":"c890d6dd-5edd-4cab-b2d8-8b869dc00db5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   de c8 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.683124196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3286","req_id":"c890d6dd-5edd-4cab-b2d8-8b869dc00db5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 57032,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 57032,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:26.695402191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3286","req_id":"c890d6dd-5edd-4cab-b2d8-8b869dc00db5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:26.713185972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:26.713915581Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.714721662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":11,"elapsed":"756.265055ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.736957173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:26.737996748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.738965702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":11,"elapsed":"875.78µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.740617433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.741253245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.790051822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-5110-4000-8000-000000000001","resource_type":"Silo","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.79100202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.791736729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.796556972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.804672711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.880404029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.881765193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.882389543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.905041832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.942341072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.967746318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:26.974292941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.975317385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:26.981053347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.042661386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.091821983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.108156565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.111238666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.121116464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.131797949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":11,"elapsed":"590.892731ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.135208683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.293251419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.295255639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.296434029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.301268481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.333833778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.365525631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.368666219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.376354939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.377028757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.460201054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.461398593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":46,"elapsed":"848.349872ms"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.49112406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":20,"elapsed":"889.124174ms"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.492037985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.492691633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.493290182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.494122004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.507728869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":11,"elapsed":"955.063735ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.530834578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.643711451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.645576385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.646385853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.647091354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.647827807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.648417217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.656542084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.657492491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.658205127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.658778987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:27.659678473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3812","req_id":"810d5424-ab2a-4792-97bd-6e5ed33b2aa4","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   3b 27 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ;'..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.662518043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3812","req_id":"810d5424-ab2a-4792-97bd-6e5ed33b2aa4","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 15143,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 15143,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:27.667773981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:3812","req_id":"810d5424-ab2a-4792-97bd-6e5ed33b2aa4","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:27.904689976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.91366937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":11,"elapsed":"1.175225965s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:27.945393897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.200984962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName(\"oxide-services\") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName(\"nexus\") }","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:28.202574259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.203323246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.214289326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.256828352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.346661818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:28.347902442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.348597643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:28.354118047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:29.995569945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:30.006903515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.01187602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":11,"elapsed":"3.354804688s"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:30.014667354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:38837"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.028524846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:30.029076472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.029461757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.029833676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.030523776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.030898641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.041821565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.117785165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":20,"elapsed":"2.479492967s"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.140943709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"a8139bf7-1871-45b1-9669-8320bf1075e6","remote_addr":"[::1]:38837","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.141719039Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"a8139bf7-1871-45b1-9669-8320bf1075e6","remote_addr":"[::1]:38837","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:30.161823599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"a8139bf7-1871-45b1-9669-8320bf1075e6","remote_addr":"[::1]:38837","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":20960,"response_code":204}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.163852398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.215533611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.21634875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.216953872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.217561238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.218128956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.221896198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"resolved names for firewall rules","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.228263895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","subnet_networks":"{Name(\"nexus\"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })], Name(\"external-dns\"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })]}","subnet_interfaces":"{}","vpc_interfaces":"{}","instance_interfaces":"{}"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.229246306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.229854143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.230471088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.231048234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.250337765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.285940184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"a8139bf7-1871-45b1-9669-8320bf1075e6\", \"date\": \"Wed, 04 Mar 2026 07:11:30 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.28651704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:30.286863448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.287235557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":6,"elapsed":"10.746831631s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.332791084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.399670482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.400689436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.401570566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.409252313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.493908447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.576163241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.576841068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.577437993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.583939876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"fetched allowlist from DB","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:30.759580553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","allowed":"AllowList { id: 001de000-a110-4000-8000-000000000000, time_created: 2026-03-04T05:46:29.930974Z, time_modified: 2026-03-04T05:46:29.930974Z, allowed_ips: None }"}
{"msg":"Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified.","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.760696861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"resolved firewall rules for sled agents","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.761347463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_agent_rules":"[]"}
{"msg":"resolved 0 rules for sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:30.761988476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:32.481882627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:32.509070505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:32.609647288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:58186"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:32.735603869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":47,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:32.736905965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:32.738315815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:32.739255972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:32.739978838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:32.761821349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:32.866531822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"d0f57003-2688-4e6f-a7a2-4bae4f2406e1","remote_addr":"[::1]:58186","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:32.868009106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"d0f57003-2688-4e6f-a7a2-4bae4f2406e1","remote_addr":"[::1]:58186","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:32.89717422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.075496824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:33.076766114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.077406667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.083684445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:33.110353271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"d0f57003-2688-4e6f-a7a2-4bae4f2406e1","remote_addr":"[::1]:58186","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":243736,"response_code":200}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:33.390391931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52643","req_id":"dc88d0d6-f908-40be-8a0e-2c2df12e2185","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   c7 1a 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.391173203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52643","req_id":"dc88d0d6-f908-40be-8a0e-2c2df12e2185","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50970,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50970,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:33.395814731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52643","req_id":"dc88d0d6-f908-40be-8a0e-2c2df12e2185","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:33.396217772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61494","req_id":"edb64c88-b659-489a-ae28-3a92ea99665e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   4c 51 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   LQ..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.396684108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61494","req_id":"edb64c88-b659-489a-ae28-3a92ea99665e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 19537,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 19537,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:33.399119244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61494","req_id":"edb64c88-b659-489a-ae28-3a92ea99665e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:33.399531453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61174","req_id":"a89d7086-d979-46f8-b049-e9146f499afe","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   14 1b 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.400277294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61174","req_id":"a89d7086-d979-46f8-b049-e9146f499afe","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 5147,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 5147,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:33.40280246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61174","req_id":"a89d7086-d979-46f8-b049-e9146f499afe","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.461107568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0f57003-2688-4e6f-a7a2-4bae4f2406e1\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:11:33 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:33.507816075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.508650923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.579789741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.755908878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:33.767433958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":47,"elapsed":"1.032239152s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.022237168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.022868151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.084292014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"bb8b129d-f6ec-49fa-acde-9e9077ef674a","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.085092965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"bb8b129d-f6ec-49fa-acde-9e9077ef674a","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:34.102438545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.318691541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.319507421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.320232882Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.326075797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.428452475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.429231402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.429955791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.430638457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.434035864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:34.443535151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.444220322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.447777126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"bf26457e-8d93-4cd3-897f-664c445595d6","remote_addr":"[::1]:58186","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.448665801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"bf26457e-8d93-4cd3-897f-664c445595d6","remote_addr":"[::1]:58186","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:34.452793959Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"bf26457e-8d93-4cd3-897f-664c445595d6","remote_addr":"[::1]:58186","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":4985,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.46176144Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"bf26457e-8d93-4cd3-897f-664c445595d6\", \"date\": \"Wed, 04 Mar 2026 07:11:34 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.462710034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":13,"elapsed":"4.43407073s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.46352869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:34.464138531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.464677301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.46537669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.466097822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.467085894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.474939192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.580394745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.604781018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.628737688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.701587494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.702603462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.703340746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.704348728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.70505453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.724220806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.783704275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.784518301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.7879111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.851027103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.852800816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:34.854027202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.855130493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.867363328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:34.897741833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.018843711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:35.019461117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.019829048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.031509817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"resolved sleds for vpc oxide-services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.059361974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vpc_to_sled":"[]"}
{"msg":"sending firewall rules to sled agents","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.05992385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"updated firewall rules on 0 sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:35.060533711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"successfully propagated service firewall rules","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.060855149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"service-firewall-rule-progator","background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","elapsed":"19.426646619s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.061171748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_firewall_rule_propagation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":2,"elapsed":"19.430087453s"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.089940215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.185328884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:35.541384029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"bb8b129d-f6ec-49fa-acde-9e9077ef674a","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":1457194,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.553637696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb8b129d-f6ec-49fa-acde-9e9077ef674a\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:35 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 15.041446401s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:35.55619239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb8b129d-f6ec-49fa-acde-9e9077ef674a\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bb8b129d-f6ec-49fa-acde-9e9077ef674a\" }"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:35.700155202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:35.700699924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:35.701333312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:35.701725611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":18,"elapsed":"1.382106778s"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:38.676749904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.677790561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:38.681471712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:52516"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.68630427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":48,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:38.686648003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:38.687424675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:38.687787335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.688279944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.695756022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:38.70459155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"0ea8e6bc-336f-4b7f-8fe9-c0acaf1c18bc","remote_addr":"[::1]:52516","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.705388914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"0ea8e6bc-336f-4b7f-8fe9-c0acaf1c18bc","remote_addr":"[::1]:52516","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:38.714712782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"0ea8e6bc-336f-4b7f-8fe9-c0acaf1c18bc","remote_addr":"[::1]:52516","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":10034,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.722859051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ea8e6bc-336f-4b7f-8fe9-c0acaf1c18bc\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:11:38 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:38.724047391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.724781649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.727588814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.734160456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.810671523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:38.811479347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.812000442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.818405413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.979868652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:38.980837597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":48,"elapsed":"293.843497ms"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:39.087722429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:39.08851354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:39.091839526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"2acc3a03-4611-4440-9631-6f3213e6835c","remote_addr":"[::1]:52516","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:39.092927356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"2acc3a03-4611-4440-9631-6f3213e6835c","remote_addr":"[::1]:52516","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:39.096301087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"2acc3a03-4611-4440-9631-6f3213e6835c","remote_addr":"[::1]:52516","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":4043,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:39.100275731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"2acc3a03-4611-4440-9631-6f3213e6835c\", \"date\": \"Wed, 04 Mar 2026 07:11:39 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:39.101346517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":14,"elapsed":"4.638618778s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.310756926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.311705169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.312097468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.322875725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.48704075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.487576003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.497464372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.670499023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.696440315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.697467565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.698645895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.706433943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.74104825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.915445486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.91661056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.917840724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:40.918773306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.919658634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.924511472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:40.981236322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:41.095478647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:41.096548783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:41.097147872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:41.114921432Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.654763466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:42.660532133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:56626"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.669437139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":21,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.671858999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.673098751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.674000752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.674770121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.681704032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.695922531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":21,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.696877768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.697768287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.698605449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.699596257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.704927295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.721687372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"f1518c6e-0e23-45b9-ac8b-62ddbb4962ea","remote_addr":"[::1]:56626","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.722863207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"f1518c6e-0e23-45b9-ac8b-62ddbb4962ea","remote_addr":"[::1]:56626","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:42.7569469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"f1518c6e-0e23-45b9-ac8b-62ddbb4962ea","remote_addr":"[::1]:56626","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":35739,"response_code":200}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.766164217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.82759341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.828405102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.828915526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.832117104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.839940673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.936238308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:42.937059318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.937860149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.94368135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.978118265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f1518c6e-0e23-45b9-ac8b-62ddbb4962ea\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:11:42 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:42.978841331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:42.979209011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:42.979542804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.979871988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":10,"elapsed":"2.66877667s"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:42.984853291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":21,"elapsed":"315.285754ms"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.016653679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":21,"elapsed":"320.450461ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.597265072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58577","req_id":"785a6e2a-5127-44ba-b6a8-f7664d1c5f82","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   8a 8b 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.598305428Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58577","req_id":"785a6e2a-5127-44ba-b6a8-f7664d1c5f82","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 35467,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 35467,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:43.601496496Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58577","req_id":"785a6e2a-5127-44ba-b6a8-f7664d1c5f82","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.6018864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50527","req_id":"364d80ff-650e-4e7c-8ed3-03c1da42d692","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   7f fa 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.602399669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50527","req_id":"364d80ff-650e-4e7c-8ed3-03c1da42d692","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32762,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32762,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:43.606084167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50527","req_id":"364d80ff-650e-4e7c-8ed3-03c1da42d692","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.606401076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65478","req_id":"e95c415b-809a-4f3d-bc8c-d4da21368af8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   e2 b6 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.607729004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65478","req_id":"e95c415b-809a-4f3d-bc8c-d4da21368af8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 58038,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 58038,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:43.613868828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65478","req_id":"e95c415b-809a-4f3d-bc8c-d4da21368af8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.688163243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":49,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.688943823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.689687099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.690742826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.691454049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.696683485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.717021798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.78822838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:43.788853181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.78925001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.79331343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.812762638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:43.813460934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":49,"elapsed":"125.517188ms"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:44.312570698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"3a651be2-11a0-4080-9e87-ad56ac473af8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:44.313252512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"3a651be2-11a0-4080-9e87-ad56ac473af8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:44.328524403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"3a651be2-11a0-4080-9e87-ad56ac473af8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:44.329755839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"3a651be2-11a0-4080-9e87-ad56ac473af8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:44.337789274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"3a651be2-11a0-4080-9e87-ad56ac473af8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:44.455340611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"3a651be2-11a0-4080-9e87-ad56ac473af8","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":142633,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.691206028Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":50,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:48.691904014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:48.692298097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:48.692669745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.693012515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.698106623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.733634845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.792029191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:48.794030235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.794507062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.798088635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.82037104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:48.821290065Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":50,"elapsed":"129.393987ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.319997389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.320655236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.321219827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.32730356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.392800522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.393476886Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.393886539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.394248127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.39800062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.533058827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.557483075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.576103447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.576801063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.582426996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.63828238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.640469984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.641054545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.641524749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.645419024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.668866592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.694346187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:49.748805668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:49.749383295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:49.749664854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:49.749905633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":19,"elapsed":"429.405976ms"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:50.601113351Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:50.601905064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:50.604935793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0fede4e3-4591-4fbb-bc68-38769d8097db","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:50.606682722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0fede4e3-4591-4fbb-bc68-38769d8097db","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:50.608973262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:50.648894168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"0fede4e3-4591-4fbb-bc68-38769d8097db","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":44175,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:50.655674564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0fede4e3-4591-4fbb-bc68-38769d8097db\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:50 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 13.834015966s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:50.656919787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0fede4e3-4591-4fbb-bc68-38769d8097db\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:11:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0fede4e3-4591-4fbb-bc68-38769d8097db\" }"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.614551147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61159","req_id":"982b8c77-4dd1-40d4-81ce-86c5068ea188","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   0b 14 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.615330254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61159","req_id":"982b8c77-4dd1-40d4-81ce-86c5068ea188","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 2836,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 2836,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:53.620506585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61159","req_id":"982b8c77-4dd1-40d4-81ce-86c5068ea188","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.621013101Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57455","req_id":"07834b3e-813e-4f94-b219-b695c75c1a19","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   99 02 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.621488765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57455","req_id":"07834b3e-813e-4f94-b219-b695c75c1a19","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 39170,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 39170,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:53.624486168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57455","req_id":"07834b3e-813e-4f94-b219-b695c75c1a19","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.629568422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55906","req_id":"41bc2b47-707e-4257-8014-608e74164ee5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   2b b8 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   +............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.630333752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55906","req_id":"41bc2b47-707e-4257-8014-608e74164ee5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 11192,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 11192,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:53.634228507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55906","req_id":"41bc2b47-707e-4257-8014-608e74164ee5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.693872616Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":51,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.694477527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.694950526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.69542564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.695870934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.699621533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.718006297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.780768578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:53.78184225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.782251042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.787233357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.808690844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:53.809335384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":51,"elapsed":"115.434372ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.624645288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:55.62541643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.625924269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.630529054Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:55.65040531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:55.656852976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30516","req_id":"dd68b52c-7b8b-4837-b144-d1b453be1452","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   03 c3 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.659531205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30516","req_id":"dd68b52c-7b8b-4837-b144-d1b453be1452","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 963,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 963,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:55.662313771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30516","req_id":"dd68b52c-7b8b-4837-b144-d1b453be1452","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:55.663991123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:55.664588319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.664963143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":12,"elapsed":"40.065774ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.956073532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:55.956681149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.957115251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:55.963826249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:55.991211639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:55.998613179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:17191","req_id":"574a3c12-b851-47cc-8ab3-0630aa008d87","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   c3 f0 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.004649648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:17191","req_id":"574a3c12-b851-47cc-8ab3-0630aa008d87","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 50160,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 50160,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:56.011436618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:17191","req_id":"574a3c12-b851-47cc-8ab3-0630aa008d87","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:56.011868295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:56.012302087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.012689706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":12,"elapsed":"57.085597ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.553732091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.559992956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.560973133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":12,"elapsed":"1.06227ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.562095482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:11:56.563649528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.564441982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":12,"elapsed":"2.349897ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.565365266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.567432862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.570595784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.571890666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.572751245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.578380435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.629894486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":12,"elapsed":"76.142215ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.656644534Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.706329723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.706959063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.707349458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.712787799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.723486387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.726356217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.726859336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.727244221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.72806943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.732197949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.745821236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:56.746365537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.746972483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":12,"elapsed":"615.673µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.747411425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.747725548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.766429416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":12,"elapsed":"200.845205ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.805600783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.883567401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.884485584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.884937492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.892803755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:56.914282423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:37009","req_id":"59e37498-64a7-4675-b55c-611519571296","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   ec 02 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.916736688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:37009","req_id":"59e37498-64a7-4675-b55c-611519571296","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 60418,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 60418,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:56.924326832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:37009","req_id":"59e37498-64a7-4675-b55c-611519571296","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:11:56.928381104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:56.931212307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":12,"elapsed":"183.684602ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.687899661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":22,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.689479319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.690715003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.691399693Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.691785549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.69523019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:11:57.700532401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.701437308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":12,"elapsed":"977.564493ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.701878695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":22,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.70239003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.702759584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.703441769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.70381003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.729535834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.807736557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.886370531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.887073136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.887486767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.919295071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.92580981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.988946375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:57.989630704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.990087982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:57.996842968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.017106854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":22,"elapsed":"329.432701ms"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.047147427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":22,"elapsed":"345.017714ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.797487504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":52,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:58.798147033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:58.79869892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:58.799105327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.799475552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.804042442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.84837491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.934345144Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:11:58.935107578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:58.935799242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:59.041781442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:59.19313873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:11:59.194114427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":52,"elapsed":"396.089416ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.029904205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:00.031359185Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.031957253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.032706511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.033462422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.034180679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.039379814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.062136912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.093934174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.095436217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.096200345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.097106955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.097806625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.098427218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.163602902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.223870949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:00.225397721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.226815327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:00.269399603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:01.480552001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.481807354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:01.484528529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:60219"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:01.500629667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"adf6539c-487a-43c3-a159-7383b46e6cde","remote_addr":"[::1]:60219","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.501206363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"adf6539c-487a-43c3-a159-7383b46e6cde","remote_addr":"[::1]:60219","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:01.573544021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"adf6539c-487a-43c3-a159-7383b46e6cde","remote_addr":"[::1]:60219","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":72719,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.583375357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"adf6539c-487a-43c3-a159-7383b46e6cde\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:12:01 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:01.58402131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.58441893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.593006034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:01.798609028Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.799275783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:01.802810202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"82beb0cf-aeac-49df-a116-d252f84d0692","remote_addr":"[::1]:60219","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.803613939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"82beb0cf-aeac-49df-a116-d252f84d0692","remote_addr":"[::1]:60219","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:01.810038588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"82beb0cf-aeac-49df-a116-d252f84d0692","remote_addr":"[::1]:60219","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":7644,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.815843598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"82beb0cf-aeac-49df-a116-d252f84d0692\", \"date\": \"Wed, 04 Mar 2026 07:12:01 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:01.816584249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":15,"elapsed":"1.786842666s"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.647050305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59218","req_id":"083067b1-7b5c-4177-ba34-c207dca91ef5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   24 0d 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   $............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.647801857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59218","req_id":"083067b1-7b5c-4177-ba34-c207dca91ef5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 9229,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 9229,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:03.652100325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59218","req_id":"083067b1-7b5c-4177-ba34-c207dca91ef5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.652492574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53971","req_id":"df955a38-da55-4ee8-afc9-57a3352b0946","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   af f3 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.652950553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53971","req_id":"df955a38-da55-4ee8-afc9-57a3352b0946","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45043,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45043,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:03.655811435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53971","req_id":"df955a38-da55-4ee8-afc9-57a3352b0946","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.656104457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58140","req_id":"46f6c645-349d-4aa7-bdc1-48a5d5b4f850","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   97 b8 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.656561785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58140","req_id":"46f6c645-349d-4aa7-bdc1-48a5d5b4f850","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 38840,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 38840,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:03.662512193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58140","req_id":"46f6c645-349d-4aa7-bdc1-48a5d5b4f850","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.818402444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":53,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.819678879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.820612834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.821290821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.821716326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.825463297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.858447747Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.927878392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:03.928566529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.929217492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.934744669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.965663547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:03.966351814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":53,"elapsed":"147.709503ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.352740043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.353342309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.353736151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.357498954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.444538763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.445311698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.445681803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.445986708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.486192811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.494605539Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.495948717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.541277884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"562d9e71-6832-41de-b4b6-68469c4304d0","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.54187503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"562d9e71-6832-41de-b4b6-68469c4304d0","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:04.542747904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.5499158Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.602599093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.623161562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.623806393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.628804299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.660980313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.703488087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:04.714722139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.715639391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.72227536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:04.746947421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"562d9e71-6832-41de-b4b6-68469c4304d0","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":206375,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.770076648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"562d9e71-6832-41de-b4b6-68469c4304d0\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:12:04 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 15.863534064s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:04.770761929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"562d9e71-6832-41de-b4b6-68469c4304d0\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:12:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"562d9e71-6832-41de-b4b6-68469c4304d0\" }"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.840673118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:04.937129528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:05.0187184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:05.019454021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:05.01979056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:05.020074283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":20,"elapsed":"666.978995ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:08.86021109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":54,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:08.897131117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:08.898159218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:08.899228512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:08.899734377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:08.911610754Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:08.941704072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:09.05423601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:09.054896281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:09.055312007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:09.060096129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:09.111804576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:09.138994457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":54,"elapsed":"278.722436ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.384556767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":11,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.385835076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.386452562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.392332199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.598916343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.599530873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.604523819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.717312146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.718033269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.718566999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.719056551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.755823924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.892788928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.980093271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.981135962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.981869228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:10.982603246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.983287585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:10.987575742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:11.031792881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:11.124994948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:11.12561049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:11.125973471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:11.132420446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.21535336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:12.239222406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:59234"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.332059318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"48cf36c0-8f2e-463c-a3a9-eeffbde2e1ab","remote_addr":"[::1]:59234","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.33299709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"48cf36c0-8f2e-463c-a3a9-eeffbde2e1ab","remote_addr":"[::1]:59234","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:12.339905601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"48cf36c0-8f2e-463c-a3a9-eeffbde2e1ab","remote_addr":"[::1]:59234","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":7982,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.345372749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"48cf36c0-8f2e-463c-a3a9-eeffbde2e1ab\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:12:12 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:12.346970593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:12.347560685Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:12.348043954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.348725628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":11,"elapsed":"1.963837875s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.690829485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":23,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.691701797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.692328312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.692722454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.693078682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.697687756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.885944439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":23,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.886674038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.887184512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:12.887595639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.887980192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:12.931720634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.031663359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.114841217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.115739892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.116133013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.1189713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.172122773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.390887237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.391624782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.391985599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.394013997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.495321462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":23,"elapsed":"804.565173ms"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.501014619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":23,"elapsed":"615.783667ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.766349737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54616","req_id":"c08255bd-cbe6-4253-a48e-9aedee17f84f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   b7 0e 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.780905906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54616","req_id":"c08255bd-cbe6-4253-a48e-9aedee17f84f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 46862,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 46862,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:13.784893737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54616","req_id":"c08255bd-cbe6-4253-a48e-9aedee17f84f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.785981096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58780","req_id":"03a0c4f1-6eb8-4dbf-9ffd-76759f4e65cb","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   eb c4 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.787154326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58780","req_id":"03a0c4f1-6eb8-4dbf-9ffd-76759f4e65cb","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 60356,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 60356,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:13.792317411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58780","req_id":"03a0c4f1-6eb8-4dbf-9ffd-76759f4e65cb","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.793771779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54938","req_id":"630e94f2-dd18-428a-997b-9500dcf78e7f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   1e ba 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.795082893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54938","req_id":"630e94f2-dd18-428a-997b-9500dcf78e7f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 7866,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 7866,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:13.799886473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54938","req_id":"630e94f2-dd18-428a-997b-9500dcf78e7f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.891991942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":55,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.892469541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.892797392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:13.893106366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.893397915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.897013666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:13.991620563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.088450816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:14.105524776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.106497148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.112355022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.222433968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.22322564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":55,"elapsed":"330.613806ms"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:14.536230924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c9513a77-ac0a-4097-a511-925a20c63f55","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.536944041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c9513a77-ac0a-4097-a511-925a20c63f55","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:14.537254277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c9513a77-ac0a-4097-a511-925a20c63f55","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.537623711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c9513a77-ac0a-4097-a511-925a20c63f55","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:14.548841049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c9513a77-ac0a-4097-a511-925a20c63f55","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:14.598051255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"c9513a77-ac0a-4097-a511-925a20c63f55","remote_addr":"127.0.0.1:40564","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":61442,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:17.981740268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:17.982342344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:17.982665305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:17.983162854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:17.987064192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.01400082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.015614345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.016687136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.017418248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.022538717Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.056578573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:18.057437309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.058330062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":7,"elapsed":"507.949µs"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.059567459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.428663825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.577479135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.578341147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.578597997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.578854296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.579253479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.579553395Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.584428727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.624988011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.625617161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.625875023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:18.626146763Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.626399555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":7,"elapsed":"178.264µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.6266688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:18.626903145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 07:02:18.626292502 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.627136709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.627411384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.630065666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.650391555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.650921598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.651380579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.6516829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.65819851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.724033634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42322","req_id":"6534ac2e-f441-41ae-a865-4bdbcca346e4","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   d8 b5 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   ............._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.724618015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42322","req_id":"6534ac2e-f441-41ae-a865-4bdbcca346e4","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55477,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55477,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:18.72746038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42322","req_id":"6534ac2e-f441-41ae-a865-4bdbcca346e4","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.727762119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:14770","req_id":"9b25a468-b86f-4239-8957-1f6b55b6b459","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   d6 08 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   ............._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.728121954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:14770","req_id":"9b25a468-b86f-4239-8957-1f6b55b6b459","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 54792,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 54792,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:18.730084251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:14770","req_id":"9b25a468-b86f-4239-8957-1f6b55b6b459","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.731471166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.936829424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:18.938147632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 07:02:18.626292502 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.938594019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T07:02:18.626292502Z\",\"pruned\":[]}","iteration":7,"elapsed":"311.876503ms"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:18.959960254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.961902702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":7,"elapsed":"384.587463ms"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:18.963594932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.964053824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":7,"elapsed":"338.62128ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.980609194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":56,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.981040471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.981346368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:18.98163944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:18.981911551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.031491561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:19.201357839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.2133015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":7,"elapsed":"1.231507125s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.264270108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.304706308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.310805782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.311626021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.312255601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.317373917Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.322471962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:19.345719934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.404671507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":7,"elapsed":"82.173263ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.454871719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.489100209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.489627797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.493140925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.496175181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.496574264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.496885051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.52360557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.529384958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.530509832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.531401764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.535844738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.548252551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.597430102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.602683786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.603795965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.610387556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.643763023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.645280466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.647057155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.652136653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.658131279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":21,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.664111817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.664758742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.667863638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.671063484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.671490522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.671724206Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.671944934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.672154951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.6723608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.672586649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.672871474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.673163585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.673432248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.67716902Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.681375644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:19.681773484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.68200263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.682210092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":7,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:19.682443565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.682648061Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.682914641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.683189517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.683447159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.686192301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.732782032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.76635456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.767154007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.76762887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.770806862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.835401422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":7,"elapsed":"338.491973ms"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:19.896227517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.897576526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":7,"elapsed":"1.31895358s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.901789764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.902314997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.902659943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.90333332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.903606312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.911821608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.912481299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":56,"elapsed":"931.463473ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.918409903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62881","req_id":"28ed13b5-a11e-4bf1-9003-62155c5b188b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   83 a5 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.918984354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62881","req_id":"28ed13b5-a11e-4bf1-9003-62155c5b188b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33701,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33701,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.921890366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62881","req_id":"28ed13b5-a11e-4bf1-9003-62155c5b188b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.922283196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:12148","req_id":"0e6d5a67-1307-4344-ad33-ecf1abb4cba7","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   78 29 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   x)..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.922710364Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:12148","req_id":"0e6d5a67-1307-4344-ad33-ecf1abb4cba7","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 30761,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 30761,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.92559809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:12148","req_id":"0e6d5a67-1307-4344-ad33-ecf1abb4cba7","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.932973778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.933606755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.933949727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.93441955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":7,"elapsed":"262.50688ms"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.934837139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:19.935303545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:19.935614774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.93593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:19.938574772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.055503142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.056230787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.05660471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.056947971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.065035564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:20.074771691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.075908919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":7,"elapsed":"432.148963ms"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.098961734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.101314627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.10173403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.107063585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.115493528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.158549721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.182827919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.238646169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.251551941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.268547966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.273783124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.281536553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.282807387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":7,"elapsed":"753.20653ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.284294651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.285088908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.289580619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.47483536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.475664978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.476053941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.476382964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.480490342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.526540531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:20.527036506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.527524736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":7,"elapsed":"1.071923357s"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.698077458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.699549662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:20.704504622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6e8ee17e-5422-40fb-b29e-49d03da3ebee","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.705028934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6e8ee17e-5422-40fb-b29e-49d03da3ebee","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:20.722179466Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.779448517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:20.798936842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.178922415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.318182567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.318751367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.319253725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.31955823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.31981517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.321947234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.339915391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.383397069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.387678512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.389310143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.416136853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.596841507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.636885247Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.657026359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.663993607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.668827277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:21.770038821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.770925963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":7,"elapsed":"2.506818759s"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:21.965852052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.966270243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:21.966501302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.966704425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":21,"elapsed":"2.308191738s"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.967736375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.968682474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:21.968956418Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.969212556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:21.981450622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:22.034506626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:22.070948553Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:22.08406333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:22.084969088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:22.220511657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:22.292491205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6e8ee17e-5422-40fb-b29e-49d03da3ebee","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":1588755,"response_code":500}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.268029394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.274929548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.275399291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":7,"elapsed":"3.593931954s"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:23.276347304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.277162733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":7,"elapsed":"3.604570313s"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:23.300219997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:62420"}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.303397628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6e8ee17e-5422-40fb-b29e-49d03da3ebee\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:12:22 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 21.457669419s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:23.303835297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6e8ee17e-5422-40fb-b29e-49d03da3ebee\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:12:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6e8ee17e-5422-40fb-b29e-49d03da3ebee\" }"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.334007642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"5f817a29-99c5-455d-bc66-020fbd2389fb","remote_addr":"[::1]:62420","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.334685578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"5f817a29-99c5-455d-bc66-020fbd2389fb","remote_addr":"[::1]:62420","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:23.339793373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"5f817a29-99c5-455d-bc66-020fbd2389fb","remote_addr":"[::1]:62420","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":6204,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.359265476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"5f817a29-99c5-455d-bc66-020fbd2389fb\", \"date\": \"Wed, 04 Mar 2026 07:12:23 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.359742784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:23.36022398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.360473646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":7,"elapsed":"3.678067805s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.4153375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:23.415761171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.416287857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.416582302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.416856136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.417314146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.448403492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.622411606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.739092173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.739567608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.739850329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.740126488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.740380682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.743925393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.794822178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.851421939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:23.854695461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.855947017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:23.863747148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:24.939635721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:24.940094462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:24.950256725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:50718"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:24.987889547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":57,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:24.988430973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:24.988762501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:24.989078619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:24.989371781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:24.992793777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:24.996474117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49661","req_id":"59f0688e-a584-4389-82ae-a183c0953036","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   c8 54 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .T..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:24.997566396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49661","req_id":"59f0688e-a584-4389-82ae-a183c0953036","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 51284,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 51284,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:24.999986944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49661","req_id":"59f0688e-a584-4389-82ae-a183c0953036","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.000505985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51508","req_id":"78114ad7-8643-44aa-af5a-cbb49460d564","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   6f f2 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   o............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.00091669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51508","req_id":"78114ad7-8643-44aa-af5a-cbb49460d564","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 28658,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 28658,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:25.005252582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51508","req_id":"78114ad7-8643-44aa-af5a-cbb49460d564","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.005608829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52783","req_id":"175854b6-7940-458b-ae04-d421df399abc","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   9b 7c 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .|..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.006579687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52783","req_id":"175854b6-7940-458b-ae04-d421df399abc","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 39804,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 39804,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:25.011414239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52783","req_id":"175854b6-7940-458b-ae04-d421df399abc","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.104756966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"c7d85992-7817-4826-b1b4-320bcd39460c","remote_addr":"[::1]:50718","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.105572064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"c7d85992-7817-4826-b1b4-320bcd39460c","remote_addr":"[::1]:50718","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:25.110014167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"c7d85992-7817-4826-b1b4-320bcd39460c","remote_addr":"[::1]:50718","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":5356,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.166212311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7d85992-7817-4826-b1b4-320bcd39460c\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:12:25 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.17175042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.172447204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.177893491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.18551112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.240837272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.241570489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.242134709Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.246563024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.266015448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.266951437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":57,"elapsed":"279.392767ms"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:25.362327953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.363314923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.367859008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"f8bd6458-8612-4c22-9e6d-646bd23c1fd7","remote_addr":"[::1]:50718","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.368740308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"f8bd6458-8612-4c22-9e6d-646bd23c1fd7","remote_addr":"[::1]:50718","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:25.379973308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"f8bd6458-8612-4c22-9e6d-646bd23c1fd7","remote_addr":"[::1]:50718","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":12189,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.390360718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"f8bd6458-8612-4c22-9e6d-646bd23c1fd7\", \"date\": \"Wed, 04 Mar 2026 07:12:25 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.39199814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":16,"elapsed":"1.976706572s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.626252369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.626806199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.627289749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.630606647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.64241059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.647640718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28941","req_id":"ef52fd2c-4bd7-4f41-8f30-318caa8a4aeb","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   ac d0 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.649187119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28941","req_id":"ef52fd2c-4bd7-4f41-8f30-318caa8a4aeb","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 44240,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 44240,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:25.652134593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28941","req_id":"ef52fd2c-4bd7-4f41-8f30-318caa8a4aeb","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:25.653194378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:25.65351285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.654014597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":13,"elapsed":"27.831846ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.954798428Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.955751551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.956302205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.961943689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.983506114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:25.99519045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55855","req_id":"fb446a80-c2d2-4adb-a24b-4697e840b73c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   d1 14 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:25.996008815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55855","req_id":"fb446a80-c2d2-4adb-a24b-4697e840b73c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 53524,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 53524,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:25.999950153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55855","req_id":"fb446a80-c2d2-4adb-a24b-4697e840b73c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:26.00746077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:26.007946184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.008219196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":13,"elapsed":"53.427581ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.58742146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.590321409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.590880189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":13,"elapsed":"258.163µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.59116235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:26.591400543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.591606141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":13,"elapsed":"99.548µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.591852981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.592077297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.592344638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.592867476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.593147693Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.596306967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.61574845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":13,"elapsed":"28.360195ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.618811593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.640192578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.640932668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.641431989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.642760137Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.657717103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":13,"elapsed":"66.590334ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.724400852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.724784444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.725219318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.725475527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.725699942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.729877139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.744489811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.763411952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.763807377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.764220447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.765477434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.770084093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:26.77064635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.770931596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":13,"elapsed":"649.961µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.771526357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.771943857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:26.773844672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:36279","req_id":"f6ede5e4-b22c-49fe-abc6-b4557cab0c09","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   db ae 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.774361939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:36279","req_id":"f6ede5e4-b22c-49fe-abc6-b4557cab0c09","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56238,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56238,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:26.776677578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:36279","req_id":"f6ede5e4-b22c-49fe-abc6-b4557cab0c09","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:26.776913346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.77727255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":13,"elapsed":"5.625552ms"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:26.81783527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:26.81897827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":13,"elapsed":"94.599191ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.699553405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":24,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.699949131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.70040681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.700713449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.700983966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.71051715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.729343621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.768885775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.769817014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.770496414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.773847339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.80245212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":24,"elapsed":"103.077891ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.914929769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":24,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.915421095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.915660912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:27.9158819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.916211465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:27.919294278Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:28.051138161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:28.115560328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:28.116011775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:28.116252503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:28.11901499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:28.215800904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":24,"elapsed":"300.951805ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:29.99861515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":58,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:29.999292806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:29.999902015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.000703547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.001230143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.007988316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.024110345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.057232699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.057652453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.057904183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.060348397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.065320412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:30.066112655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.066687327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.067255435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.067796349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.068294008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.071741575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.08148203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.082236579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":58,"elapsed":"83.693513ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.087228764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.125827604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.126363038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.127022167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.127494114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.127862245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.130428051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.188448244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.213286645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.213745977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.213977978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.216776847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:30.899449621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.899941909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:30.921204518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:34093"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.935394841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"300e7d36-5b4d-4c12-af7c-a94357a5806e","remote_addr":"[::1]:34093","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.935809765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"300e7d36-5b4d-4c12-af7c-a94357a5806e","remote_addr":"[::1]:34093","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:30.938409829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"300e7d36-5b4d-4c12-af7c-a94357a5806e","remote_addr":"[::1]:34093","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":3139,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.941691527Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"300e7d36-5b4d-4c12-af7c-a94357a5806e\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:12:30 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:30.942285727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.942546986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:30.948284952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:31.254352988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:31.255358435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:31.262245303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"64b2a4f2-a313-4d68-8402-f28c935ae64c","remote_addr":"[::1]:34093","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:31.263642067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"64b2a4f2-a313-4d68-8402-f28c935ae64c","remote_addr":"[::1]:34093","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:31.267725557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"64b2a4f2-a313-4d68-8402-f28c935ae64c","remote_addr":"[::1]:34093","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":5661,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:31.272845386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"64b2a4f2-a313-4d68-8402-f28c935ae64c\", \"date\": \"Wed, 04 Mar 2026 07:12:31 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:31.273504976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":17,"elapsed":"1.208116489s"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.659544789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":22,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.660000905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.660264268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.665279889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.672735867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.673034941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.674104997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.674362047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.682083903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.724578651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.755793487Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.766145817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.766560641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.780871032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.78737003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.788051393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:34.788388603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.788587207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.791045991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.856316563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:34.879225132Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.022022559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":59,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.022512923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.02272271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.022917476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.023094688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.026608076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.106352295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.1463937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.14680623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.14701803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.149151157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:35.224941844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.225328822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:35.225657044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.2258113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":22,"elapsed":"566.040643ms"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.232471608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.232841293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":59,"elapsed":"210.683956ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.235480314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59845","req_id":"eb4318a5-20df-4e73-b0c4-0f34a01706ce","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   5e ec 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ^............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.235843094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59845","req_id":"eb4318a5-20df-4e73-b0c4-0f34a01706ce","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 24300,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 24300,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:35.237170992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59845","req_id":"eb4318a5-20df-4e73-b0c4-0f34a01706ce","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.237317012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57818","req_id":"24c8b292-3ee1-4521-884e-debb886963ea","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   53 e9 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   S............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.237541698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57818","req_id":"24c8b292-3ee1-4521-884e-debb886963ea","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 21481,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 21481,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:35.241909833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57818","req_id":"24c8b292-3ee1-4521-884e-debb886963ea","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:35.2779138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54025","req_id":"ce197504-c4cc-4b6e-95de-26db6576fb15","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   8e 2f 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ./..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:35.278409024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54025","req_id":"ce197504-c4cc-4b6e-95de-26db6576fb15","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 36399,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 36399,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:35.280690415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54025","req_id":"ce197504-c4cc-4b6e-95de-26db6576fb15","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.051714141Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":60,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.052058034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.052257139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.0524436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.052687995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.054540384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.075021401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.145875303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.146268103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.146431016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.148224228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.227422823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.227822537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":60,"elapsed":"175.724934ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.401320197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":12,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.401751524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.401937223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.404968423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.462549043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.462879139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.465328774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.503901552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.504311427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.504502987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.504665139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.507820476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.524642977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.537903593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.538276083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.538459598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.538633463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.538791246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.539808867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.591814505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.606444321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:40.606893092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.607256003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:40.608333954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:41.20192289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:41.215746405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:60362"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:41.219945655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"c727a2cc-52dc-46e6-a238-ee1d8bb3c490","remote_addr":"[::1]:60362","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:41.228264767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"c727a2cc-52dc-46e6-a238-ee1d8bb3c490","remote_addr":"[::1]:60362","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:41.231504522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"c727a2cc-52dc-46e6-a238-ee1d8bb3c490","remote_addr":"[::1]:60362","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":11520,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:41.234446966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c727a2cc-52dc-46e6-a238-ee1d8bb3c490\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:12:41 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:41.234723495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:41.234867892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:41.235002038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:41.235135283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":12,"elapsed":"833.894493ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.723126946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":25,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.723402744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.72369254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.723845744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.723984449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.726332533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.747670121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.768328891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.768573617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.768750488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.770522778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.803680492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":25,"elapsed":"80.582293ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.923246731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":25,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.923553269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.923712967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.923864317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.924003393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.925521348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.943131714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.961081794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:42.961364676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.961518341Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.963507081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:42.986699353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":25,"elapsed":"63.541008ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:44.636571194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:50199"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:44.63782277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"32300e36-0057-4b6d-a523-c030bd940c04","remote_addr":"127.0.0.1:50199","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.638301881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"32300e36-0057-4b6d-a523-c030bd940c04","remote_addr":"127.0.0.1:50199","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:44.639356546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"32300e36-0057-4b6d-a523-c030bd940c04","remote_addr":"127.0.0.1:50199","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.639961077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"32300e36-0057-4b6d-a523-c030bd940c04","remote_addr":"127.0.0.1:50199","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.643982113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"32300e36-0057-4b6d-a523-c030bd940c04","remote_addr":"127.0.0.1:50199","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:44.664460464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"32300e36-0057-4b6d-a523-c030bd940c04","remote_addr":"127.0.0.1:50199","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":26655,"response_code":200}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.763690624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.764557245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:44.765814833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"33c970f0-791a-48fc-a68b-a2c31afd6b18","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.766123215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"33c970f0-791a-48fc-a68b-a2c31afd6b18","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:44.766954777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:44.78097414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"33c970f0-791a-48fc-a68b-a2c31afd6b18","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":15296,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:44.783554245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"33c970f0-791a-48fc-a68b-a2c31afd6b18\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:12:44 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 22.829236953s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:44.7841712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"33c970f0-791a-48fc-a68b-a2c31afd6b18\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:12:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"33c970f0-791a-48fc-a68b-a2c31afd6b18\" }"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.053631781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":61,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.053913871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.054065162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.05420538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.054335238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.055736942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.065296458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.099401984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.099623444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.099750677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.10416276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.115732378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.11600009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":61,"elapsed":"62.468417ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.250913739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60705","req_id":"67d0588e-b71a-4f22-81b7-c5ea115c5065","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   ae 29 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .)..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.251441798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60705","req_id":"67d0588e-b71a-4f22-81b7-c5ea115c5065","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 44585,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 44585,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:45.253885271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60705","req_id":"67d0588e-b71a-4f22-81b7-c5ea115c5065","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.256293454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52808","req_id":"235b179f-7fc7-4ef7-97be-9207498723af","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   5f ed 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   _............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.256884598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52808","req_id":"235b179f-7fc7-4ef7-97be-9207498723af","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 24557,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 24557,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:45.259939886Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52808","req_id":"235b179f-7fc7-4ef7-97be-9207498723af","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:45.333758646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63074","req_id":"a3e5ea05-c317-4c54-aef4-ffba9717f48a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   ca 78 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .x..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:45.334169221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63074","req_id":"a3e5ea05-c317-4c54-aef4-ffba9717f48a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 51832,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 51832,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:45.335521237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63074","req_id":"a3e5ea05-c317-4c54-aef4-ffba9717f48a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.659017567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":23,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.659253505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.659367492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.662218975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.666143299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.666321723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.666435429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.66653635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.667909537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.674813829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.683956047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.688643877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.688886228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.690422008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.693055839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.693223663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.693337449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.693436936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.694582552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.699984971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.708050981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:49.719644888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:49.719816869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:49.719911036Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:49.719989983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":23,"elapsed":"60.946104ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.055491737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":62,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:50.055828336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:50.056114514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:50.056371184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.056598104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.05971782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.067833449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.080250781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:50.080520266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.080635585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.081279744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.086085429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:50.086273562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":62,"elapsed":"30.671486ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.053787603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":63,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.053964975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.054049122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.054125013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.054194792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.055575273Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.060766413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.075958666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.076117131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.076198252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.076260416Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.07928342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.079546602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":63,"elapsed":"25.886373ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.261245051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61364","req_id":"2d44bb3a-aa81-435e-828a-bbf7d5137f51","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   84 46 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .F..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.261417032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61364","req_id":"2d44bb3a-aa81-435e-828a-bbf7d5137f51","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33862,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33862,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.262082414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61364","req_id":"2d44bb3a-aa81-435e-828a-bbf7d5137f51","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.262801762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60975","req_id":"f7ac28c0-96be-4d12-be1c-4ee19514f25b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   b3 38 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .8..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.263263089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60975","req_id":"f7ac28c0-96be-4d12-be1c-4ee19514f25b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45880,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45880,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.263751098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60975","req_id":"f7ac28c0-96be-4d12-be1c-4ee19514f25b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.337988509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49569","req_id":"3951bc19-63e2-4413-9ff8-e9510a2711ed","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   d0 6c 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .l..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.338178016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49569","req_id":"3951bc19-63e2-4413-9ff8-e9510a2711ed","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 53356,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 53356,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.338948947Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49569","req_id":"3951bc19-63e2-4413-9ff8-e9510a2711ed","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.6244873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.624626276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.624707698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.625859014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.630570321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.631690135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28839","req_id":"6bd6ba77-aad0-43dd-8140-cddf02f2e875","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   12 00 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.631882567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28839","req_id":"6bd6ba77-aad0-43dd-8140-cddf02f2e875","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 4608,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 4608,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.632597016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:28839","req_id":"6bd6ba77-aad0-43dd-8140-cddf02f2e875","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.632822164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.632919136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.632985758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":14,"elapsed":"8.509631ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.680390889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":5,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.680570906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.680664843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.68074896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.680828077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.681936439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.687645157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.702809484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.702974021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.703050433Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.704111931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read target blueprint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:55.709552707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: no target blueprint set"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.709886119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read target blueprint: Internal Error: no target blueprint set\"}","iteration":5,"elapsed":"29.260203ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.954812352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.955009644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.955099753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.955998448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.961534844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:55.962901969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:32823","req_id":"6a23b47a-08d0-48e3-aa3a-22162a43f815","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   e6 7b 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .{..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.963098038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:32823","req_id":"6a23b47a-08d0-48e3-aa3a-22162a43f815","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 59003,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 59003,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.963537591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:32823","req_id":"6a23b47a-08d0-48e3-aa3a-22162a43f815","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.963808479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:55.963922586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:55.963991864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":14,"elapsed":"9.20382ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.588710857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.591322374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.591457182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":14,"elapsed":"101.973µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.591530567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:12:56.591589013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.59163778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":14,"elapsed":"39.879µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.591696697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.591814211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.591926033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.591995821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.592059057Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.593343177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.594465887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":14,"elapsed":"5.782685ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.597785109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.603778102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.603923771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.604011986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.604639032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.61049373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":14,"elapsed":"18.786141ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.72426032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.724409526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.724492821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.724571357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.72464301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.725467267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.730751512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.737068999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.737322232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.737475316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.73781501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:56.764358708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.764493276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":14,"elapsed":"40.350399ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.77145914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:12:56.771611713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.771683576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":14,"elapsed":"244.175µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.771773084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.771838434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:56.772786026Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24053","req_id":"6f7288ed-e890-43dc-ad29-9f5b0591a0c8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   34 66 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   4f..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.772915773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24053","req_id":"6f7288ed-e890-43dc-ad29-9f5b0591a0c8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 13414,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 13414,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:56.77359385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24053","req_id":"6f7288ed-e890-43dc-ad29-9f5b0591a0c8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:12:56.773806993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:56.774086819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":14,"elapsed":"2.316ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.72355972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":26,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.723695981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.72377629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.723851991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.723922481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.72505479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.73007979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.73774339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.737968347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.738138004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.738515203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.742678672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":26,"elapsed":"19.080165ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.923806367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":26,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.923986234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.924082936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.924176231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.924263274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.92527815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.930147541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.941105875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:12:57.941262285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.941333858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.942379104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:12:57.946988819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":26,"elapsed":"23.178424ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.05325143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":64,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.053408712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.053522909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.053694079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.053805119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.054591301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.059396765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.066874516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.066975186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.067036989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.067812589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.068781153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:00.068869549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.068910911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.068962934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.069013064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.069084265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.069717142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.07214817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.072243469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":64,"elapsed":"19.015296ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.073950911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.080940242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.081031193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.081086963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.081141191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.081189457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.081810821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.086581867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.093127777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.093280561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.093392022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.093916674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:00.192430731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.192559988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:00.193370317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:60682"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.195222987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"e98498b5-5134-40b7-835c-ac259d2c5196","remote_addr":"[::1]:60682","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.195311573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"e98498b5-5134-40b7-835c-ac259d2c5196","remote_addr":"[::1]:60682","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:00.195503043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"e98498b5-5134-40b7-835c-ac259d2c5196","remote_addr":"[::1]:60682","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":317,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.196277862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e98498b5-5134-40b7-835c-ac259d2c5196\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:13:00 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.196357149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.19652336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.197367907Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:00.222108885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.222295635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:00.223302195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"4fd18607-dacd-4f52-b0cf-fc481b60d31b","remote_addr":"[::1]:60682","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.223393456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"4fd18607-dacd-4f52-b0cf-fc481b60d31b","remote_addr":"[::1]:60682","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:00.223664304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"4fd18607-dacd-4f52-b0cf-fc481b60d31b","remote_addr":"[::1]:60682","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":378,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.224172483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"4fd18607-dacd-4f52-b0cf-fc481b60d31b\", \"date\": \"Wed, 04 Mar 2026 07:13:00 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:00.224420796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":18,"elapsed":"155.664983ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.658935067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":24,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.659072961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.659121728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.6599334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.661844465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.661913422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.66195751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.661994804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.662393025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.665408824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.668030762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.670535166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.670597339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.671085328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.672472103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.672541871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.672585859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.672624886Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.673023257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.675903648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.678386649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:04.684055809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:04.684150437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:04.684188192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:04.684218522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":24,"elapsed":"25.243215ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.053438513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":65,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.053562309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.053612309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.053657408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.053698329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.054140488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.057606551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.062606231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.062688184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.062732973Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.063062668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.065768943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.065847098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":65,"elapsed":"12.414677ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.269123021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60319","req_id":"5c578945-527c-4dc3-a8c9-0b124532e086","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   d4 a7 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.269367997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60319","req_id":"5c578945-527c-4dc3-a8c9-0b124532e086","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 54439,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 54439,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:05.270099821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60319","req_id":"5c578945-527c-4dc3-a8c9-0b124532e086","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.270281552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55900","req_id":"ac9476b6-29e5-4fc1-a836-e6a82b7f86df","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   9d 3b 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .;..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.270477992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55900","req_id":"ac9476b6-29e5-4fc1-a836-e6a82b7f86df","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 40251,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 40251,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:05.271074647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55900","req_id":"ac9476b6-29e5-4fc1-a836-e6a82b7f86df","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:05.34060525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59166","req_id":"8e9ad2df-f80c-4bb0-b305-53e3ff301f06","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   3c 9e 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   <............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:05.340734056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59166","req_id":"8e9ad2df-f80c-4bb0-b305-53e3ff301f06","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 15518,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 15518,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:05.341125774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59166","req_id":"8e9ad2df-f80c-4bb0-b305-53e3ff301f06","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:07.615401413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:07.615687982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:07.616293325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"ba174456-5fb0-40e1-b1e4-ac7afd479835","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:07.616378394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"ba174456-5fb0-40e1-b1e4-ac7afd479835","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:07.616888697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:07.62354124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"ba174456-5fb0-40e1-b1e4-ac7afd479835","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":7253,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:07.62405493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba174456-5fb0-40e1-b1e4-ac7afd479835\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:13:07 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 29.003411811s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:07.624141261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba174456-5fb0-40e1-b1e4-ac7afd479835\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:13:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ba174456-5fb0-40e1-b1e4-ac7afd479835\" }"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.053384588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":66,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.05348126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.053530037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.053573774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.053615186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.054319625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.058075885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.063103841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.063266645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.063313818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.063518946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.067321859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.067471786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":66,"elapsed":"14.004014ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.40217787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":13,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.402302067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.402351235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.403616918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.416387031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.416479254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.417506765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.42769046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.427805639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.427852472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.42789151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.42839029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.431162396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.435115988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.435268521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.435316426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.435360614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.435399541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.435672994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.438017912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.441187978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.441258929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.441303818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.441674103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.4986438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:10.499284673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:56570"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:10.499686982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d2358334-ee89-4ff6-9277-374074876a81","remote_addr":"[::1]:56570","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.499757161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d2358334-ee89-4ff6-9277-374074876a81","remote_addr":"[::1]:56570","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:10.499955375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d2358334-ee89-4ff6-9277-374074876a81","remote_addr":"[::1]:56570","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":275,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.500234248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2358334-ee89-4ff6-9277-374074876a81\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:13:10 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:10.500298967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:10.500334588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:10.500367643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:10.500400789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":13,"elapsed":"98.336494ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.72377844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":27,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.72388402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.723932296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.723976153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.724017415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.72474014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.727554099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.7322627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.732344944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.732390494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.73269571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.73520297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":27,"elapsed":"11.464269ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.923906862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":27,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.92402204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.924110366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.924172229Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.924222529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.924786839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.928955569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.933257954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:12.933345077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.933401178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.933758147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:12.936695701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":27,"elapsed":"12.83403ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:14.701423146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:60344"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:14.701708272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"ba5fa5d5-69a3-418d-aa4c-cf6ec98c69ab","remote_addr":"127.0.0.1:60344","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:14.70181268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"ba5fa5d5-69a3-418d-aa4c-cf6ec98c69ab","remote_addr":"127.0.0.1:60344","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:14.701959932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"ba5fa5d5-69a3-418d-aa4c-cf6ec98c69ab","remote_addr":"127.0.0.1:60344","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:14.702034771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"ba5fa5d5-69a3-418d-aa4c-cf6ec98c69ab","remote_addr":"127.0.0.1:60344","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:14.702778197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"ba5fa5d5-69a3-418d-aa4c-cf6ec98c69ab","remote_addr":"127.0.0.1:60344","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:14.706788642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"ba5fa5d5-69a3-418d-aa4c-cf6ec98c69ab","remote_addr":"127.0.0.1:60344","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":5123,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.053319852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":67,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.053401333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.05344514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.05348517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.053522855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.054149159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.05805696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.062946351Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.063033704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.06307689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.063378018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.065834627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.065918474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":67,"elapsed":"12.463684ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.27395915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59291","req_id":"a95abb66-0c19-40db-849e-08ba47f08c9c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   78 5d 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   x]..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.274091583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59291","req_id":"a95abb66-0c19-40db-849e-08ba47f08c9c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 30813,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 30813,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:15.274512398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59291","req_id":"a95abb66-0c19-40db-849e-08ba47f08c9c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.27455841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53000","req_id":"e9b50262-3517-4980-ac84-f741d19c72b2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   22 82 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   \"............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.274608559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53000","req_id":"e9b50262-3517-4980-ac84-f741d19c72b2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 8834,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 8834,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:15.27486056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53000","req_id":"e9b50262-3517-4980-ac84-f741d19c72b2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:15.342875764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65193","req_id":"b4a53eb9-eb1c-4f5a-bc54-f161f8d7bf62","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   db e7 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:15.343024739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65193","req_id":"b4a53eb9-eb1c-4f5a-bc54-f161f8d7bf62","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56295,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56295,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:15.343334214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65193","req_id":"b4a53eb9-eb1c-4f5a-bc54-f161f8d7bf62","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.980683858Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.980867272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:17.980914696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.980964735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.981624626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:17.983767672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:17.983848793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:17.98390272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.983950205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.984445619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.987609643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.993361647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:17.99483847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:17.994913409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":8,"elapsed":"14.287346ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.057100006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:18.057220906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.057264323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":8,"elapsed":"116.662µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.577645863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.577743107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.578374611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55492","req_id":"74d9f9da-9c45-497b-9ecb-5e6d195ede5b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   18 20 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   . ..........._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.578451223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55492","req_id":"74d9f9da-9c45-497b-9ecb-5e6d195ede5b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6176,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6176,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:18.578920134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55492","req_id":"74d9f9da-9c45-497b-9ecb-5e6d195ede5b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:18.579120823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.579973777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":8,"elapsed":"2.336871ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.580092883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.580144185Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.580177602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.580217611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.581247777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.582985839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.583066559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.583112731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.583151438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.583638245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.586840986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.593807051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:18.595323302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.595417008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":8,"elapsed":"15.143096ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.626426837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.626510292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.626995025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:18.627064263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.627101627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":8,"elapsed":"78.666µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.627138821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:18.627169021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 07:03:18.627104382 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.62719883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.627265252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.628070331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:18.629226998Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62484","req_id":"b4ad7a14-88d1-49a1-a83d-50ff379e8bc5","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   91 06 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   ............._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.629305173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62484","req_id":"b4ad7a14-88d1-49a1-a83d-50ff379e8bc5","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 37126,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 37126,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:18.629771319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62484","req_id":"b4ad7a14-88d1-49a1-a83d-50ff379e8bc5","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:18.62991284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.629968951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":8,"elapsed":"3.574258ms"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:18.632044904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 07:03:18.627104382 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:18.635967946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T07:03:18.627104382Z\",\"pruned\":[]}","iteration":8,"elapsed":"8.794326ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.264934125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.265040115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.265088271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.265131116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.26516853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.266344626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.269764348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.273333236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.273427012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.27347128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.273814442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.277711351Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.277845558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.277933392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.278074492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.278170814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.280969652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.286977374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.287093405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.287135659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.287494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:19.29056544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.290630089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":8,"elapsed":"25.693129ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.322877755Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.32297079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.323006221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":8,"elapsed":"106.872µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.456358469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.45653493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.456604588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.457199149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.462487963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.462599384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.462643842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":8,"elapsed":"6.322747ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.496987051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.497124794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.497276626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.497997738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.502532515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":8,"elapsed":"5.511707ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.530344231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.530486133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.530533637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.531199139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.538543736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.538622162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.538755346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.539622328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.542565183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.542755912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":8,"elapsed":"12.517039ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.644977492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.645068423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.64521292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.645962178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.649439805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.649554563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":8,"elapsed":"4.544235ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.659398744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":25,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.659543281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.659588511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.660239754Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.661757118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.661826836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.661871996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.661911074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.662426267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.665200607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.667585313Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.670319564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.670392819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.671103731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.672237372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.672302622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.672336299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.672368473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.672398492Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.672701204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.672759289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.672803777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.672929096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.672978444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.673291847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.673915105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.673969823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.674010083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.674046936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.67440627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.675148424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:16839","req_id":"4c91ec11-0b94-4983-8554-73aa0fed49ee","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   a3 47 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .G..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.675237271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:16839","req_id":"4c91ec11-0b94-4983-8554-73aa0fed49ee","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 41799,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 41799,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.67565468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:16839","req_id":"4c91ec11-0b94-4983-8554-73aa0fed49ee","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.675836641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.67589682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.675937892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.675978974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":8,"elapsed":"3.760158ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.67609298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.679574725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.679642209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.679685044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.680014719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.680525123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.682729611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.683157751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.683215967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.683248241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.695030001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":8,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.695117805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.695170119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.695228785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.695293133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.695352762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.695779219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.697000183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30318","req_id":"a19bfb95-886c-4548-9ad4-b2c6988cc3b3","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   01 b3 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.697127386Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30318","req_id":"a19bfb95-886c-4548-9ad4-b2c6988cc3b3","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 435,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 435,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.697729702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30318","req_id":"a19bfb95-886c-4548-9ad4-b2c6988cc3b3","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.69780919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:19.697880291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.697943196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.698003005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.6983024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.700858036Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.715938046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.716007774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.716050599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.71613653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.716184305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.717942106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.721771401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.725426671Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.72549178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.725531259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.725957746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.72777272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.727839593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.727874593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.727904051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":25,"elapsed":"68.596148ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.729760488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.729874986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.72992246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.729959944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.730332314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.733021445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.737257839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.737362306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.737405722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.737871257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.738970059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.739067683Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":8,"elapsed":"66.359085ms"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.818471405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.81899289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.819052348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":8,"elapsed":"135.894768ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.820067414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:35843"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.82050343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"fc82e08f-86b7-4e05-b130-f634c8e6fc19","remote_addr":"[::1]:35843","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.820574622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"fc82e08f-86b7-4e05-b130-f634c8e6fc19","remote_addr":"[::1]:35843","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.820810951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"fc82e08f-86b7-4e05-b130-f634c8e6fc19","remote_addr":"[::1]:35843","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":323,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.821351956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"fc82e08f-86b7-4e05-b130-f634c8e6fc19\", \"date\": \"Wed, 04 Mar 2026 07:13:19 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.821473447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.821521923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.821554398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":8,"elapsed":"126.557664ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.82168667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.821739175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.821768874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.821807771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.82184769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.821884423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.822504394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.833848124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.83953573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.839610218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.839716649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.839770877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.839810365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.840279688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.84358904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.848915719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.849029716Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.849074405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.849608515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.92931056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.929422222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.93155107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:49374"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.933579679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"081650b1-2161-4337-8220-f8a3f64cd73b","remote_addr":"[::1]:49374","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.933693245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"081650b1-2161-4337-8220-f8a3f64cd73b","remote_addr":"[::1]:49374","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.933740438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"081650b1-2161-4337-8220-f8a3f64cd73b","remote_addr":"[::1]:49374","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":290,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.933789937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"081650b1-2161-4337-8220-f8a3f64cd73b\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:13:19 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.933842611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.933923552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.934371361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.970983606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.971114266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:19.971654118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"6147b4d9-fd0a-4569-9f7d-ce1c258f76fc","remote_addr":"[::1]:49374","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.971774778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"6147b4d9-fd0a-4569-9f7d-ce1c258f76fc","remote_addr":"[::1]:49374","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:19.97197719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"6147b4d9-fd0a-4569-9f7d-ce1c258f76fc","remote_addr":"[::1]:49374","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":347,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.972443406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"6147b4d9-fd0a-4569-9f7d-ce1c258f76fc\", \"date\": \"Wed, 04 Mar 2026 07:13:19 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:19.97261659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":19,"elapsed":"150.916142ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.058750118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":68,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:20.058870828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:20.058937931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:20.058999483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.059059242Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.059903809Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.068183654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.07324706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:20.073326267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.073369984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.073686403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.080790692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:20.080867595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":68,"elapsed":"22.147286ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.060072446Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":69,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.060284718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.060374025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.060449034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.060514233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.060948766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.064474699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.069452245Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.069517655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.069560109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.06991778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.073580524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.073676555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":69,"elapsed":"13.616343ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.277272824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54668","req_id":"725948df-1301-4de2-a831-40a49d234985","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   1a ee 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.277426319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54668","req_id":"725948df-1301-4de2-a831-40a49d234985","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6894,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6894,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.277909218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54668","req_id":"725948df-1301-4de2-a831-40a49d234985","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.277955069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50983","req_id":"62fdbbd9-d15a-49c3-8667-ed52d696211c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   68 be 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   h............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.2780058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50983","req_id":"62fdbbd9-d15a-49c3-8667-ed52d696211c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26814,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26814,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.278324803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50983","req_id":"62fdbbd9-d15a-49c3-8667-ed52d696211c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.345271976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59026","req_id":"d0bef0a2-f6c7-4c99-9ce8-dc1f22910976","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   1e ff 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.345405441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59026","req_id":"d0bef0a2-f6c7-4c99-9ce8-dc1f22910976","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 7935,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 7935,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.345788752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59026","req_id":"d0bef0a2-f6c7-4c99-9ce8-dc1f22910976","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.624385559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.624504204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.624553713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.625382929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.629282073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.629920461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57139","req_id":"00a748c8-dfed-4b91-99d1-1581e847df63","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   89 3c 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .<..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.630003226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57139","req_id":"00a748c8-dfed-4b91-99d1-1581e847df63","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 35132,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 35132,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.630430675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57139","req_id":"00a748c8-dfed-4b91-99d1-1581e847df63","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.630627696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.630691032Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.630729198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":15,"elapsed":"6.383117ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.953626238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.953756857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.953802889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.954442649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.979056384Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:25.979832826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7079","req_id":"069c9f06-850e-4f80-9f53-d451e581f6d2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   80 70 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .p..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.979942294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7079","req_id":"069c9f06-850e-4f80-9f53-d451e581f6d2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32880,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32880,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.980427838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:7079","req_id":"069c9f06-850e-4f80-9f53-d451e581f6d2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.980588547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:25.980654989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:25.98069586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":15,"elapsed":"27.164301ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.588303389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.591200283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.591278027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":15,"elapsed":"72.003µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.592574161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:26.592637216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.59266899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":15,"elapsed":"62.504µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.592706805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.592741073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.592782004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.592821853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.5928607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.593521322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.594395268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":15,"elapsed":"6.124224ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.596793732Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.600258202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.600331598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.600374844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.600747624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.60435546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":15,"elapsed":"11.630509ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.723651642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.723752362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.723798544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.72384225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.723880066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.724517932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.727760863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.731219953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.731292437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.731334942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.731679316Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:26.739988368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.740137434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":15,"elapsed":"16.43424ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.77145332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:26.771519992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.7715491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":15,"elapsed":"140.409µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.771592005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.771623879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:26.772211987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5298","req_id":"8db002b8-d1bd-4170-88a8-421134d8166f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   98 17 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.772281405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5298","req_id":"8db002b8-d1bd-4170-88a8-421134d8166f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 38935,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 38935,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:26.772723262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:5298","req_id":"8db002b8-d1bd-4170-88a8-421134d8166f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:26.772874974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:26.772944792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":15,"elapsed":"1.328479ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.723633177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":28,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.723753777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.723798766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.723841841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.723881109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.724610658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.728737244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.733816863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.733963654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.734010397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.734297477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.737869381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":28,"elapsed":"14.273298ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.924455988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":28,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.924554784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.924607298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.924659722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.924706004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.925366997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.928806608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.93267935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:27.932767215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.932821863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.93325819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:27.936729514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":28,"elapsed":"12.340349ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.059950926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":70,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.060080764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.060130843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.060178268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.060221123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.060841555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.064543978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.069435643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.069537185Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.069583708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.069941558Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.070564896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:30.07063239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.070667851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.070711628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.070755415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.070797178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.07107016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.072845877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.072991806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":70,"elapsed":"13.005239ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.073867666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.077136197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.077209122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.077254112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.077297488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.077337277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.077659978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.080831587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.084093756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.084186109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.084229465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.084546264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:30.15906773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.159240453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:30.159905995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:38798"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.167866095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"10de4ad6-a56c-4ca7-b1a4-60f09350455e","remote_addr":"[::1]:38798","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.168234015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"10de4ad6-a56c-4ca7-b1a4-60f09350455e","remote_addr":"[::1]:38798","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:30.169003964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"10de4ad6-a56c-4ca7-b1a4-60f09350455e","remote_addr":"[::1]:38798","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":1272,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.170860261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10de4ad6-a56c-4ca7-b1a4-60f09350455e\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:13:30 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.17130799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.171382979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.171993762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:30.191838406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.191913565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:30.19255554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"e78f52b7-db17-4d92-aa8e-78d37dfbfca3","remote_addr":"[::1]:38798","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.192633344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"e78f52b7-db17-4d92-aa8e-78d37dfbfca3","remote_addr":"[::1]:38798","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:30.193128808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"e78f52b7-db17-4d92-aa8e-78d37dfbfca3","remote_addr":"[::1]:38798","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":588,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.193652108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"e78f52b7-db17-4d92-aa8e-78d37dfbfca3\", \"date\": \"Wed, 04 Mar 2026 07:13:30 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:30.193826524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":20,"elapsed":"123.244895ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.659195083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":26,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.659298479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.659355392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.660061304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.661996437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.662122187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.662171304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.662208969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.662628813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.66558812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.668598148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.671510873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.671630391Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.672408286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.674482044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.674628305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.674675839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.674863753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.675397713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.680074932Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.680534124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:34.688849409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:34.688964748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:34.68900608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:34.689040338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":26,"elapsed":"29.910174ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.060371632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":71,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.060470879Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.060520338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.060565808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.060607881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.06134736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.06522414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.070191315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.070285733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.070331473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.070684414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.073852697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.073935231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":71,"elapsed":"13.59457ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.279927909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50968","req_id":"177f0403-1258-497e-b412-4e5b25f69d80","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   37 9e 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   7............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.280083458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50968","req_id":"177f0403-1258-497e-b412-4e5b25f69d80","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 14238,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 14238,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:35.280511468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50968","req_id":"177f0403-1258-497e-b412-4e5b25f69d80","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.28067867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51005","req_id":"c418fe35-2549-4b33-92f4-068ba74b8a65","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   39 26 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   9&..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.280746234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51005","req_id":"c418fe35-2549-4b33-92f4-068ba74b8a65","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 14630,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 14630,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:35.28111163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51005","req_id":"c418fe35-2549-4b33-92f4-068ba74b8a65","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:35.347648328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53500","req_id":"222f8588-5af1-460f-a1ba-22d691d1c380","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   b3 21 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .!..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:35.34781594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53500","req_id":"222f8588-5af1-460f-a1ba-22d691d1c380","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45857,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45857,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:35.348300944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53500","req_id":"222f8588-5af1-460f-a1ba-22d691d1c380","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:36.629416344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:36.629635159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:36.630250281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f2743b03-c5ac-41f1-b3dc-7a6c45caa047","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:36.630324779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f2743b03-c5ac-41f1-b3dc-7a6c45caa047","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:36.630997044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:36.786051123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"f2743b03-c5ac-41f1-b3dc-7a6c45caa047","remote_addr":"127.0.0.1:40074","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":155791,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:36.78661334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2743b03-c5ac-41f1-b3dc-7a6c45caa047\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:13:36 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 59.066645652s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:36.786691405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2743b03-c5ac-41f1-b3dc-7a6c45caa047\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:13:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2743b03-c5ac-41f1-b3dc-7a6c45caa047\" }"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.060131256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":72,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.060281094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.060333668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.060379038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.060417665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.061001164Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.064382429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.069180669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.069324805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.069371528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.069648117Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.072829325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.072913022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":72,"elapsed":"12.813448ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.401661493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":14,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.401796512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.401845148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.402685637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.417512094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.417688154Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.418373555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.484438295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.484569816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.484619094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.484659424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.485272271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.4894207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.493304424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.493380906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.493422949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.49346354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.493501485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.493869286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.497491029Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.50085497Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.500929538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.500973225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.50137857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.554286239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:40.554959506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:58144"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:40.555506372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d3c30ddc-d6c7-41cc-8d4f-51e7105e0719","remote_addr":"[::1]:58144","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.555647923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d3c30ddc-d6c7-41cc-8d4f-51e7105e0719","remote_addr":"[::1]:58144","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:40.555719926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d3c30ddc-d6c7-41cc-8d4f-51e7105e0719","remote_addr":"[::1]:58144","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":251,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.555939793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d3c30ddc-d6c7-41cc-8d4f-51e7105e0719\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:13:40 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:40.556055643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:40.556112496Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:40.556168196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:40.556286542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":14,"elapsed":"154.519358ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.723696295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":29,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.723805713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.723857836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.723902475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.723943967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.724666102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.727987438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.733015935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.73311388Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.733158719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.733492522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.736190971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":29,"elapsed":"12.524895ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.924201806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":29,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.924293708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.924348627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.924403426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.924455349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.925026053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.928376657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.932532752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:42.932618883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.932676217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.933021352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:42.935420076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":29,"elapsed":"11.233481ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:44.745436899Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:37781"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:44.745691826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"76de8ab3-968b-4e72-943f-bb3530b3c531","remote_addr":"127.0.0.1:37781","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:44.745766524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"76de8ab3-968b-4e72-943f-bb3530b3c531","remote_addr":"127.0.0.1:37781","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:44.746009126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"76de8ab3-968b-4e72-943f-bb3530b3c531","remote_addr":"127.0.0.1:37781","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:44.746092111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"76de8ab3-968b-4e72-943f-bb3530b3c531","remote_addr":"127.0.0.1:37781","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:44.746919895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"76de8ab3-968b-4e72-943f-bb3530b3c531","remote_addr":"127.0.0.1:37781","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:44.752141576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"76de8ab3-968b-4e72-943f-bb3530b3c531","remote_addr":"127.0.0.1:37781","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":6459,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.060368499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":73,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.060507946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.060559959Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.060607564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.060651771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.06145161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.06469941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.069691174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.069774791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.069821283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.070181679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.072867373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.073016759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":73,"elapsed":"12.578553ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.283006677Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52485","req_id":"5e2067a8-4e43-4251-9376-722dc724e590","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   0c 0e 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.283178829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52485","req_id":"5e2067a8-4e43-4251-9376-722dc724e590","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 3086,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 3086,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:45.283760754Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52485","req_id":"5e2067a8-4e43-4251-9376-722dc724e590","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.283842106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65486","req_id":"a2aebd88-bfe5-4c45-8552-b2567d80059b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   80 49 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .I..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.283902195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65486","req_id":"a2aebd88-bfe5-4c45-8552-b2567d80059b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32841,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32841,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:45.284188985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65486","req_id":"a2aebd88-bfe5-4c45-8552-b2567d80059b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:45.350358784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55757","req_id":"b5916c6c-98c0-48d0-8b9e-5c6e6d02be29","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   e2 6c 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .l..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:45.350465476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55757","req_id":"b5916c6c-98c0-48d0-8b9e-5c6e6d02be29","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 57964,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 57964,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:45.350874187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55757","req_id":"b5916c6c-98c0-48d0-8b9e-5c6e6d02be29","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.658903358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":27,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.659059888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.659110429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.659903193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.661711815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.661783017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.661829579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.661867956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.662334051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.66493143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.66728773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.669949707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.670156688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.67053031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.67195549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.67206118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.672110157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.67215686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.672506084Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.674694691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.677082042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:49.682108315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:49.682223864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:49.682262511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:49.6822924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":27,"elapsed":"23.341257ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.06049394Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":74,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:50.060621653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:50.060669649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:50.060712714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.060749437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.061348607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.065441546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.070322149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:50.070389273Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.070427458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.070783696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.073586342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:50.073653876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":74,"elapsed":"13.226178ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.06019148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":75,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.060314795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.06036252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.060404985Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.060443491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.061241967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.064809893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.069734714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.069878059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.069925373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.070204136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.073264424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.073343471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":75,"elapsed":"13.154215ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.285565031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50434","req_id":"351062dd-66bb-4eec-95c0-e9e2b22ab03a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   be 5b 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   .[..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.285697544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50434","req_id":"351062dd-66bb-4eec-95c0-e9e2b22ab03a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 48731,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 48731,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.286100725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50434","req_id":"351062dd-66bb-4eec-95c0-e9e2b22ab03a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.286146897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57273","req_id":"59bef386-a84d-4f96-aec0-537d99ed3244","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   d8 3f 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .?..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.28619889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57273","req_id":"59bef386-a84d-4f96-aec0-537d99ed3244","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55359,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55359,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.28649578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57273","req_id":"59bef386-a84d-4f96-aec0-537d99ed3244","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.352765687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57822","req_id":"d05331b1-d88e-4c17-8616-e61cf97cedd2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   7c ea 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   |............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.352876738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57822","req_id":"d05331b1-d88e-4c17-8616-e61cf97cedd2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 31978,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 31978,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.353359987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57822","req_id":"d05331b1-d88e-4c17-8616-e61cf97cedd2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.623680667Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.623774884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.623822057Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.624676735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.627678056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.628620518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:1112","req_id":"fe7caa16-48b0-4928-9bf9-e0d38fb075c9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   11 09 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.628834903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:1112","req_id":"fe7caa16-48b0-4928-9bf9-e0d38fb075c9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 4361,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 4361,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.629274797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:1112","req_id":"fe7caa16-48b0-4928-9bf9-e0d38fb075c9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.629658389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.629722857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.629761654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":16,"elapsed":"5.998333ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.954097241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.954184123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.954225946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.954743635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.958112776Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:55.959370995Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:11578","req_id":"7e86e9b6-4aaa-4117-8632-f6065571b1bb","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   95 72 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .r..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.959510332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:11578","req_id":"7e86e9b6-4aaa-4117-8632-f6065571b1bb","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 38258,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 38258,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.959941989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:11578","req_id":"7e86e9b6-4aaa-4117-8632-f6065571b1bb","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.960122027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:55.960171795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:55.960263838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":16,"elapsed":"6.179222ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.587906098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.590963821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.591074931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":16,"elapsed":"73.486µs"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.591468723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":16,"elapsed":"3.602655ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.591600124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:13:56.591649502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.591682658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":16,"elapsed":"39.569µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.591722347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.591756635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.591797617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.591840271Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.591881112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.592651302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.595704656Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.600169764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.600279542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.600326746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.600719987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.603600358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":16,"elapsed":"11.892991ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.723813641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.723907929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.723956786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.724047566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.724092225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.724647658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.727750691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.731507722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.731609635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.731696287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.731989108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:56.739425197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.739512921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":16,"elapsed":"15.722867ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.770495806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:13:56.770603199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.770640133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":16,"elapsed":"138.425µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.771730047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.771816188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:56.772341011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26702","req_id":"7cb8841d-7bd1-45e1-8d9c-c959d7be4f13","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   d2 d6 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.772414617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26702","req_id":"7cb8841d-7bd1-45e1-8d9c-c959d7be4f13","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 53974,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 53974,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:56.772985451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26702","req_id":"7cb8841d-7bd1-45e1-8d9c-c959d7be4f13","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:13:56.773046983Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:56.773098746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":16,"elapsed":"1.35415ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.724479235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":30,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.724566359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.724611839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.724653652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.724693912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.725466777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.729050204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.733996698Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.734138389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.734181124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.734397614Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.737046595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":30,"elapsed":"12.448935ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.924113686Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":30,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.924203013Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.924253273Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.924298092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.924340246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.925040637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.928821947Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.933296434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:13:57.933362785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.933405921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.933806927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:13:57.936770844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":30,"elapsed":"12.624984ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.059950994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":76,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.060069921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.060118848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.060164959Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.06020571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.060919819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.064142138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.068932723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.069001299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.069042361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.069351495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.072104332Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":21,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:00.072170634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.07220405Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.07224413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.072287005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.072355371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.072679254Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.073343964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.07341736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":76,"elapsed":"13.500484ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.075533633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.078917463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.078992102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.079052472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.079095637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.079134274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.079401294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.082226636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.086338352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.08641304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.086456096Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.087004325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:00.145192321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.145322269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:00.146013762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:41335"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.146747469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"be9c3acd-4884-4667-86dd-c4d97dbce8f9","remote_addr":"[::1]:41335","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.146810485Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"be9c3acd-4884-4667-86dd-c4d97dbce8f9","remote_addr":"[::1]:41335","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:00.147395547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"be9c3acd-4884-4667-86dd-c4d97dbce8f9","remote_addr":"[::1]:41335","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":631,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.15598215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be9c3acd-4884-4667-86dd-c4d97dbce8f9\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:14:00 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.156070436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.156115265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.156902348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:00.189140055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.189492465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:00.190262053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"29a0ce05-b461-4717-8602-61d822576af0","remote_addr":"[::1]:41335","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.190336912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"29a0ce05-b461-4717-8602-61d822576af0","remote_addr":"[::1]:41335","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:00.191294874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"29a0ce05-b461-4717-8602-61d822576af0","remote_addr":"[::1]:41335","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":987,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.191794236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"29a0ce05-b461-4717-8602-61d822576af0\", \"date\": \"Wed, 04 Mar 2026 07:14:00 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:00.191979193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":21,"elapsed":"119.848509ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.658900847Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":28,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.658990074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.659034953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.660033787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.662127165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.662205881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.66225072Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.662289848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.662792195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.667141644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.66993427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.672956453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.673030249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.673744678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.675473452Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.675617728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.675665062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.675704651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.675960549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.678775069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.681193312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:04.687434647Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:04.687523293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:04.687560167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:04.687593864Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":28,"elapsed":"28.676154ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.059639116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":77,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.059773182Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.059840926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.059901196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.059955925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.060551207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.064323609Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.069271086Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.069379711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.069451564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.069889734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.073293715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.073396599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":77,"elapsed":"13.77074ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.288375665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59962","req_id":"6581f4f7-2ea6-4bbc-8226-00460bb15db0","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   14 c3 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.288502738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59962","req_id":"6581f4f7-2ea6-4bbc-8226-00460bb15db0","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 5315,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 5315,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:05.288912241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59962","req_id":"6581f4f7-2ea6-4bbc-8226-00460bb15db0","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.288955457Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55311","req_id":"d298099f-facf-49c6-9c90-8910b18b808c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   4e a4 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   N............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.28900723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55311","req_id":"d298099f-facf-49c6-9c90-8910b18b808c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 20132,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 20132,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:05.289293258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55311","req_id":"d298099f-facf-49c6-9c90-8910b18b808c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:05.355281877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60527","req_id":"fd8d74d6-5da3-4645-9ec8-c78d1d0536b2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   0d 94 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:05.355415312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60527","req_id":"fd8d74d6-5da3-4645-9ec8-c78d1d0536b2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 3476,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 3476,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:05.355845777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60527","req_id":"fd8d74d6-5da3-4645-9ec8-c78d1d0536b2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.060186694Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":78,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.060294729Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.060342784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.060386521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.060427933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.061183915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.065080404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.070011728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.070137067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.070185994Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.070456792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.073601547Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.073671266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":78,"elapsed":"13.517146ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.402291593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":15,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.40243588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.402517582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.403208684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.415335489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.415470227Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.415943577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.425744663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.425851836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.425901544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.425942876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.426593699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.429627695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.433386931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.433449936Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.433491569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.433532059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.433570175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.433932925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.437696119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.441577308Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.44171955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.441767896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.442120486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.499990611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:10.500701283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:63668"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:10.501237307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"98ab5c18-6f2f-4a3c-81e2-84381a4363f4","remote_addr":"[::1]:63668","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.501311124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"98ab5c18-6f2f-4a3c-81e2-84381a4363f4","remote_addr":"[::1]:63668","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:10.501475871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"98ab5c18-6f2f-4a3c-81e2-84381a4363f4","remote_addr":"[::1]:63668","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":261,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.50177216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98ab5c18-6f2f-4a3c-81e2-84381a4363f4\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:14:10 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:10.501844052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:10.501879443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:10.501914843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:10.50194855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":15,"elapsed":"99.689021ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.724511634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":31,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.724615961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.724662774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.72470623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.72474662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.725601408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.729401175Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.734463198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.734592595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.73464044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.73493731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.738098078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":31,"elapsed":"13.596905ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.924271062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":31,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.924427843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.924477712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.924522411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.92456202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.925419954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.931967668Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.93697345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:12.93707891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.937132626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.937663911Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:12.941225695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":31,"elapsed":"17.005454ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:14.786203603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:40643"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:14.786477857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6b7d2068-436e-486c-9998-5493b22baa9f","remote_addr":"127.0.0.1:40643","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:14.786545451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6b7d2068-436e-486c-9998-5493b22baa9f","remote_addr":"127.0.0.1:40643","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:14.786792322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6b7d2068-436e-486c-9998-5493b22baa9f","remote_addr":"127.0.0.1:40643","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:14.786878343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6b7d2068-436e-486c-9998-5493b22baa9f","remote_addr":"127.0.0.1:40643","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:14.787712179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6b7d2068-436e-486c-9998-5493b22baa9f","remote_addr":"127.0.0.1:40643","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:14.792607662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"6b7d2068-436e-486c-9998-5493b22baa9f","remote_addr":"127.0.0.1:40643","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":6171,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.059807963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":79,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.059886479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.059930407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.059973252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.060014173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.060701127Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.064810789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.069844787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.069946499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.069990807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.070320582Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.073544966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.073625236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":79,"elapsed":"13.671132ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.290206264Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55351","req_id":"177ee4a1-07cd-49c4-a187-90c16ce417f7","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   dc 09 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.290333366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55351","req_id":"177ee4a1-07cd-49c4-a187-90c16ce417f7","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 56329,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 56329,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:15.290721056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55351","req_id":"177ee4a1-07cd-49c4-a187-90c16ce417f7","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.291142092Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62472","req_id":"0b4fbf46-73d0-4ef5-b0a2-376856805259","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   23 89 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   #............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.291258052Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62472","req_id":"0b4fbf46-73d0-4ef5-b0a2-376856805259","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 9097,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 9097,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:15.291519041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62472","req_id":"0b4fbf46-73d0-4ef5-b0a2-376856805259","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:15.35740208Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52375","req_id":"395cb8d2-2a96-4bea-b7fe-87638dc32222","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   bf 61 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .a..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:15.357540184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52375","req_id":"395cb8d2-2a96-4bea-b7fe-87638dc32222","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 48993,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 48993,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:15.357917153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52375","req_id":"395cb8d2-2a96-4bea-b7fe-87638dc32222","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.981139719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.981347742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:17.981405306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.981463051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.982203071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:17.984077163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:17.984180909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:17.984269314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.98432736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.984748726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.987440613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:17.994165228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:18.050785872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.050880079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":9,"elapsed":"69.798565ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.056932209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:18.057110022Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.057155953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":9,"elapsed":"55.771µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.578087396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.578211954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.578876884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26447","req_id":"2e031cbf-da65-423d-82d1-2c675c4ea509","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   f2 f9 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   ............._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.578968927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26447","req_id":"2e031cbf-da65-423d-82d1-2c675c4ea509","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62201,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62201,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:18.579527136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26447","req_id":"2e031cbf-da65-423d-82d1-2c675c4ea509","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:18.57971067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.579777473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":9,"elapsed":"1.760176ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.579826851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.579862632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.579894846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.579933843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.581044098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.582916537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.582996857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.583044261Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.583083799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.583737027Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.587206618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.59275157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:18.594336719Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.594408742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":9,"elapsed":"14.515729ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.62654669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.62662708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.626917036Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:18.626973568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.627010732Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":9,"elapsed":"82.183µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.627114799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:18.627163736Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 07:04:18.627093937 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.627198505Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.627241059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.628105196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:18.6296481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54924","req_id":"18477127-f2b4-4690-92ba-9b6ef41dd190","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   66 46 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   fF..........._ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.629756305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54924","req_id":"18477127-f2b4-4690-92ba-9b6ef41dd190","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26182,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26182,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:18.630122192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54924","req_id":"18477127-f2b4-4690-92ba-9b6ef41dd190","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:18.630419262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.63048876Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":9,"elapsed":"3.981537ms"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:18.632409705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 07:04:18.627093937 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:18.632466548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T07:04:18.627093937Z\",\"pruned\":[]}","iteration":9,"elapsed":"5.35176ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.265060101Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.265188586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.265235629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.265277863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.265315097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.266074666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.26951597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.273187703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.273323573Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.273367149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.273626344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.278075841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.278154106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.278199336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.278237722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.278268934Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.283126933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.288459083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.288640674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.288691324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.289017953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.322850828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.323010986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.323047949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":9,"elapsed":"120.329µs"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:19.34135056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.341455618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":9,"elapsed":"76.391761ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.455830637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.455967108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.456014863Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.45673344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.462423761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.462573579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.462615793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":9,"elapsed":"6.707311ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.496643945Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.496773222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.496819684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.497355959Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.5016057Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":9,"elapsed":"4.979229ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.529614688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.529716931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.529763383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.530357062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.537058251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.537129513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.537176617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.537604807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.540629234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.540698091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":9,"elapsed":"11.111429ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.644792791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.644881908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.644926417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.645786855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.649517866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.649588205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":9,"elapsed":"4.875122ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.658705222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":29,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.658845491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.658969948Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.659551112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.661293062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.661366448Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.661413672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.661454703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.661987231Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.665235412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.667906106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.670305161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.670377034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.670854632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.671699931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.671765862Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.671801894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.671834077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.671864859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.67225328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49778","req_id":"82bdd066-e07e-4224-a414-9f3f89a85d47","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   da 06 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.672335343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49778","req_id":"82bdd066-e07e-4224-a414-9f3f89a85d47","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 55814,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 55814,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.672820166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49778","req_id":"82bdd066-e07e-4224-a414-9f3f89a85d47","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.67286757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.672909824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.672949513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.672986266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.673331832Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.674278422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.674349503Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.674394192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.674436075Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":9,"elapsed":"2.706506ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.674484942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.674550743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.674604901Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.674647545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.674685962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.675097739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.677472526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.67990171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.680107018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.683426992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.683537171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.683715194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.683860353Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.68461919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.684685091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.684718287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.685148551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:41805","req_id":"7191d204-ee77-471d-9cfe-26e14ae8825e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   2e 8b 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.685228099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:41805","req_id":"7191d204-ee77-471d-9cfe-26e14ae8825e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 11915,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 11915,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.685591751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:41805","req_id":"7191d204-ee77-471d-9cfe-26e14ae8825e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.685920103Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:19.686033258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.686082696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.686123648Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.6865073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.692248993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.692361837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.692433439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.692467718Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":29,"elapsed":"33.596445ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.69641626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":9,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.69648184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.696513884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.696553362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.696593181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.696631557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.697484962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.700723324Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.704536788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.704739891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.704790121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.704832655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.704869438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.705093794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.708489398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.71248803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.712563409Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.712605623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.713105626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.714113107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.714238757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":9,"elapsed":"29.571181ms"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.717197453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.717289856Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.717340266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.71738238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.717792966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.731347356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.735288042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.73537226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.735418301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.735877774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.740495124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.740572778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":9,"elapsed":"66.10516ms"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.806237283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.806812355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:45905"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.807528287Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"690df5ea-6e73-4b55-b7ec-f27295c3ed9e","remote_addr":"[::1]:45905","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.807614638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"690df5ea-6e73-4b55-b7ec-f27295c3ed9e","remote_addr":"[::1]:45905","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.807959623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"690df5ea-6e73-4b55-b7ec-f27295c3ed9e","remote_addr":"[::1]:45905","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":444,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.808275711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"690df5ea-6e73-4b55-b7ec-f27295c3ed9e\", \"date\": \"Wed, 04 Mar 2026 07:14:19 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.808338546Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.808434657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.808544114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":9,"elapsed":"112.16053ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.808823078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":22,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.808875412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.808904309Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.808939499Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.808976703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.809011172Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.809826491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.818307014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.822345124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.822407799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.822445523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.822484872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.822521364Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.822891429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.839993756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.844092767Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.844183928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.844227063Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.844663019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.92941337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.92949368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.930097639Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:45593"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.930618073Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"2405d675-a249-4b13-8ee3-c5eddf90a632","remote_addr":"[::1]:45593","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.930679555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"2405d675-a249-4b13-8ee3-c5eddf90a632","remote_addr":"[::1]:45593","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.930858921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"2405d675-a249-4b13-8ee3-c5eddf90a632","remote_addr":"[::1]:45593","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":259,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.931172705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2405d675-a249-4b13-8ee3-c5eddf90a632\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:14:19 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.931239207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.931277603Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.931977493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.959539714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.959614532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:19.960164224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"d99b3fbd-28d5-45c5-bbef-f2d1331abd99","remote_addr":"[::1]:45593","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.960235305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"d99b3fbd-28d5-45c5-bbef-f2d1331abd99","remote_addr":"[::1]:45593","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:19.960430463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"d99b3fbd-28d5-45c5-bbef-f2d1331abd99","remote_addr":"[::1]:45593","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":348,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.960704297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"d99b3fbd-28d5-45c5-bbef-f2d1331abd99\", \"date\": \"Wed, 04 Mar 2026 07:14:19 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:19.96085181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":22,"elapsed":"152.055134ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.060350753Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":80,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:20.060467495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:20.06052038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:20.060568956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.060612112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.061201633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.067633617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.080119465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:20.080218071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.080266277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.080674538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.088438898Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:20.088585489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":80,"elapsed":"28.314385ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.060490443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":81,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.060670751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.06072069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.060764056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.060803915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.061552352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.065243112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.070076612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.070183774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.070230377Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.0704634Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.073587134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.073663325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":81,"elapsed":"13.094357ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.293543976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61834","req_id":"c711f9d2-6d1b-48f7-a462-63dd51be3001","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   58 10 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   X............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.293655798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61834","req_id":"c711f9d2-6d1b-48f7-a462-63dd51be3001","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 22544,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 22544,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.294085221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61834","req_id":"c711f9d2-6d1b-48f7-a462-63dd51be3001","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.294126143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55679","req_id":"66887ae5-df49-4f74-b251-c290b200bb43","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   45 ef 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   E............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.294174909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55679","req_id":"66887ae5-df49-4f74-b251-c290b200bb43","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 17903,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 17903,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.294475937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:55679","req_id":"66887ae5-df49-4f74-b251-c290b200bb43","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.359873152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58563","req_id":"e826885d-2454-49ec-b29a-fe5774785327","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   d4 ce 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.360011576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58563","req_id":"e826885d-2454-49ec-b29a-fe5774785327","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 54478,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 54478,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.360450438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58563","req_id":"e826885d-2454-49ec-b29a-fe5774785327","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.623867306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.623961272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.624008626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.624840699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.627887721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.628659494Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53098","req_id":"5c238808-1d30-45ed-860a-6baaeeb2cb3c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   42 3e 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   B>..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.628749252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53098","req_id":"5c238808-1d30-45ed-860a-6baaeeb2cb3c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 16958,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 16958,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.629154687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:53098","req_id":"5c238808-1d30-45ed-860a-6baaeeb2cb3c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.629299996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.629357981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.629396567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":17,"elapsed":"5.528109ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.954048062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.954187779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.95423349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.954815306Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.958631184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:25.959480431Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:44264","req_id":"4d58477f-f550-4d0b-8726-41f6e36c3130","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   1b 57 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .W..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.960094651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:44264","req_id":"4d58477f-f550-4d0b-8726-41f6e36c3130","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6999,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6999,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.960425548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:44264","req_id":"4d58477f-f550-4d0b-8726-41f6e36c3130","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.960470778Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:25.960513423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:25.96055233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":17,"elapsed":"6.221015ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.588400909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.591387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.591475616Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":17,"elapsed":"60.66µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.591518471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:26.591567939Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.591599382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":17,"elapsed":"31.662µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.591649802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.59168428Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.591739951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.591783006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.591834929Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.59251008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.593577601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":17,"elapsed":"5.059789ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.595709565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.59989621Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.599977311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.600048091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.600386955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.603067358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":17,"elapsed":"11.432205ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.724296279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.724410616Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.72445795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.724501997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.724541967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.725204783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.728754824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.732789747Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.732880878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.732924235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.73337517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:26.74035846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.740422938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":17,"elapsed":"16.203692ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.77101702Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:26.771155585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.771191757Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":17,"elapsed":"149.266µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.772355759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.772498963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:26.772949759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:25249","req_id":"68f02748-751c-4eba-9cc6-0dafa7f61c43","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   69 fc 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   i............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.773032082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:25249","req_id":"68f02748-751c-4eba-9cc6-0dafa7f61c43","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 27132,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 27132,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:26.773470803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:25249","req_id":"68f02748-751c-4eba-9cc6-0dafa7f61c43","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:26.773642585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:26.773708515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":17,"elapsed":"1.465621ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.723791329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":32,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.723880987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.723927238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.723968751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.724007317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.724616497Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.727759189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.732518241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.732626045Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.732671726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.732936572Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.736124744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":32,"elapsed":"12.319107ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.923901704Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":32,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.92405118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.924107442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.924158854Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.924206349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.924760078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.927999382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.93194085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:27.932102952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.932165977Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.932384651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:27.935209522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":32,"elapsed":"11.270253ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.059699193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":82,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.059866816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.059916004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.059960853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.060000662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.061005397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.065636004Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.070471318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.070543731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.070587598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.070919758Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.071915295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":23,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:30.071985735Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.072019733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.072060273Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.072101696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.07213935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.07239095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.074165334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.074248078Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":82,"elapsed":"14.589856ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.075583861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.079563596Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.079679917Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.079749495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.079815255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.079875986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.080052687Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.083382781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.086908062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.086983292Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.087076296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.087373376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:30.142806511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.142916159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:30.143516521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:33834"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.144236111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"a836bad3-fe71-42dc-93df-4080198565b5","remote_addr":"[::1]:33834","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.144305689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"a836bad3-fe71-42dc-93df-4080198565b5","remote_addr":"[::1]:33834","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:30.144916382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"a836bad3-fe71-42dc-93df-4080198565b5","remote_addr":"[::1]:33834","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":609,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.145411244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a836bad3-fe71-42dc-93df-4080198565b5\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:14:30 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.145482095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.145526012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.146207756Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:30.167000963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.167091012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:30.167448031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"2ebab98b-ce18-4782-943f-0754bc94f25f","remote_addr":"[::1]:33834","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.167525165Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"2ebab98b-ce18-4782-943f-0754bc94f25f","remote_addr":"[::1]:33834","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:30.167914949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"2ebab98b-ce18-4782-943f-0754bc94f25f","remote_addr":"[::1]:33834","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":511,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.168261868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"2ebab98b-ce18-4782-943f-0754bc94f25f\", \"date\": \"Wed, 04 Mar 2026 07:14:30 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:30.168398619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":23,"elapsed":"96.568794ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.658554739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":30,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.658657644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.658704657Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.659415619Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.661111467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.661183069Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.66122869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.661268519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.661689295Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.664462913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.666954993Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.669651498Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.669724834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.670102504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.671264001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.671335533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.671381474Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.671420191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.671808903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.674311333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.676748363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:34.682628171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:34.682702699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:34.682740875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:34.682771696Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":30,"elapsed":"24.211666ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.059942789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":83,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.060126774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.060177635Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.060249178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.060298856Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.060774892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.064239673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.0690565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.06916731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.069211988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.069518607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.071980076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.072058111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":83,"elapsed":"12.150001ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.297167407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52877","req_id":"c5b62f94-6b3e-4a6b-abc2-6782ed8fa16f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   32 a0 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   2............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.297320751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52877","req_id":"c5b62f94-6b3e-4a6b-abc2-6782ed8fa16f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 12960,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 12960,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:35.297739623Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52877","req_id":"c5b62f94-6b3e-4a6b-abc2-6782ed8fa16f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.297780304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63921","req_id":"09e0f599-9de6-423a-8488-14916cd7a89b","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   18 ba 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.297859451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63921","req_id":"09e0f599-9de6-423a-8488-14916cd7a89b","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6330,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6330,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:35.29813081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63921","req_id":"09e0f599-9de6-423a-8488-14916cd7a89b","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.362341538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61277","req_id":"9ea44ed6-543b-4fcb-8755-db27ef49c73e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   f6 65 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .e..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.362454903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61277","req_id":"9ea44ed6-543b-4fcb-8755-db27ef49c73e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 63077,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 63077,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:35.362876871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:61277","req_id":"9ea44ed6-543b-4fcb-8755-db27ef49c73e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.680265251Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":6,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.680376612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.680424076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.680470048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.680510368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.681306128Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.684916869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.690114943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.69019372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.690237356Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.69061166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read target blueprint","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:35.693787607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: no target blueprint set"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.693858789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read target blueprint: Internal Error: no target blueprint set\"}","iteration":6,"elapsed":"13.686904ms"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.855085327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.855324162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:35.855905436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:54775"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:35.856370971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"e38fe1ec-208e-430a-ab0f-f33e65dce2e8","remote_addr":"127.0.0.1:54775","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.856437773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"e38fe1ec-208e-430a-ab0f-f33e65dce2e8","remote_addr":"127.0.0.1:54775","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:35.856822878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:35.86312277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"e38fe1ec-208e-430a-ab0f-f33e65dce2e8","remote_addr":"127.0.0.1:54775","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":6666,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:35.863837119Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e38fe1ec-208e-430a-ab0f-f33e65dce2e8\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:14:35 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 31.129083594s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:35.863976496Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e38fe1ec-208e-430a-ab0f-f33e65dce2e8\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:14:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e38fe1ec-208e-430a-ab0f-f33e65dce2e8\" }"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.059698666Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":84,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.059860016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.059943632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.060022389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.060127217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.060746066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.064198393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.069623548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.069737213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.069782664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.070054984Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.072681571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.072758554Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":84,"elapsed":"13.086832ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.402061407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":16,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.402214551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.402288458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.40322136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.416039919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.416139016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.41689105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.426227392Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.426308243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.426355587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.42639734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.427105877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.430812829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.434756903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.434818074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.434860148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.434902662Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.43494207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.435290522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.442217089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.447587265Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.447719818Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.447763495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.448281524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.513433581Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:40.514134202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:60691"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:40.514803722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d1a756be-2cf3-4edd-af6e-0423814854d6","remote_addr":"[::1]:60691","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.514910925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d1a756be-2cf3-4edd-af6e-0423814854d6","remote_addr":"[::1]:60691","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:40.514979942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"d1a756be-2cf3-4edd-af6e-0423814854d6","remote_addr":"[::1]:60691","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":272,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.515236802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1a756be-2cf3-4edd-af6e-0423814854d6\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:14:40 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:40.515358975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:40.515406339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:40.515439234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:40.51553777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":16,"elapsed":"113.538617ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.724174975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":33,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.724316987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.724367197Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.724412387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.724486664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.72535556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.729097362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.734103454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.734191228Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.734233533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.734689869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.738214419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":33,"elapsed":"14.121086ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.923771851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":33,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.923906739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.923959624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.924010064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.924076626Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.924657259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.928545652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.934377795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:42.934527853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.934578935Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.935087294Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:42.938789277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":33,"elapsed":"15.075251ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:44.827573577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:44240"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:44.827805147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d6622c96-21fb-4a15-ba03-2dbddb0843e6","remote_addr":"127.0.0.1:44240","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:44.82786673Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d6622c96-21fb-4a15-ba03-2dbddb0843e6","remote_addr":"127.0.0.1:44240","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:44.828179531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d6622c96-21fb-4a15-ba03-2dbddb0843e6","remote_addr":"127.0.0.1:44240","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:44.828258788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d6622c96-21fb-4a15-ba03-2dbddb0843e6","remote_addr":"127.0.0.1:44240","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:44.829131111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d6622c96-21fb-4a15-ba03-2dbddb0843e6","remote_addr":"127.0.0.1:44240","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:44.833332014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d6622c96-21fb-4a15-ba03-2dbddb0843e6","remote_addr":"127.0.0.1:44240","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":5536,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.060473286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":85,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.060605688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.060654085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.060699565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.060740176Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.061357411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.064841722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.069872203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.069948183Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.06999186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.070394299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.073129131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.073243158Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":85,"elapsed":"12.763419ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.300246415Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56628","req_id":"848b7fc1-4c7a-455d-b788-53af61a714b3","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   62 d9 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   b............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.300395611Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56628","req_id":"848b7fc1-4c7a-455d-b788-53af61a714b3","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 25305,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 25305,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:45.300679234Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56628","req_id":"848b7fc1-4c7a-455d-b788-53af61a714b3","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.3008329Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54415","req_id":"f7b99a2a-9e53-4e97-9ffa-7e847ef9be41","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   41 97 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   A............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.300903981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54415","req_id":"f7b99a2a-9e53-4e97-9ffa-7e847ef9be41","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 16791,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 16791,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:45.301187023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54415","req_id":"f7b99a2a-9e53-4e97-9ffa-7e847ef9be41","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:45.36448094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52458","req_id":"65c7cca3-a9d3-4ee2-b5d8-6f3283d7659a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   a3 62 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .b..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:45.364634104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52458","req_id":"65c7cca3-a9d3-4ee2-b5d8-6f3283d7659a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 41826,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 41826,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:45.364993147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:52458","req_id":"65c7cca3-a9d3-4ee2-b5d8-6f3283d7659a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.659496213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":31,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.659616703Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.659664257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.660400189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.66241963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.662487043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.662566281Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.662606801Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.663072777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.66661895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.669120608Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.671988023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.672090016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.672866598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.674777884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.67495184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.675181766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.675238509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.675804643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.678928427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.681836583Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:49.68798751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:49.688111867Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:49.688150333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:49.688180833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":31,"elapsed":"28.537669ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.060493346Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":86,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:50.060590018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:50.060640589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:50.060686881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.060730347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.061389697Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.065163822Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.070385303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:50.07046467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.07051006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.070873552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.073954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:50.074028999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":86,"elapsed":"13.666974ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.060110789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":87,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.06026256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.060313682Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.060356627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.060396486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.061180814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.06449198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.06965805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.069800273Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.069844491Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.0701311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.073265766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.073340825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":87,"elapsed":"13.286237ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.303486844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56787","req_id":"9bbc8645-a1a9-4d28-91f9-374a491f1cb7","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   ab 1e 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.303634878Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56787","req_id":"9bbc8645-a1a9-4d28-91f9-374a491f1cb7","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 43806,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 43806,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.304233907Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56787","req_id":"9bbc8645-a1a9-4d28-91f9-374a491f1cb7","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.304532941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51150","req_id":"d99ac0ad-c9c9-496e-a399-f1027cf71cd3","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   e2 44 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .D..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.304610706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51150","req_id":"d99ac0ad-c9c9-496e-a399-f1027cf71cd3","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 57924,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 57924,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.305150378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51150","req_id":"d99ac0ad-c9c9-496e-a399-f1027cf71cd3","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.366116181Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54285","req_id":"349008ac-7386-4a69-8f8c-f7d99be75e24","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   67 96 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   g............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.366223213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54285","req_id":"349008ac-7386-4a69-8f8c-f7d99be75e24","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26518,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26518,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.366728918Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54285","req_id":"349008ac-7386-4a69-8f8c-f7d99be75e24","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.624232601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.624333602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.624383451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.625085044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.628431891Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.630029545Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50072","req_id":"cd58f55b-6b8f-47d7-ac54-03883700b64e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   84 29 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   .)..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.630132579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50072","req_id":"cd58f55b-6b8f-47d7-ac54-03883700b64e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33833,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33833,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.630409589Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50072","req_id":"cd58f55b-6b8f-47d7-ac54-03883700b64e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.630445861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.630489889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.630528706Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":18,"elapsed":"5.842895ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.954334861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.954422645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.954466112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.955229808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.958682335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:55.959602924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24086","req_id":"eeef1b2e-322f-48cb-af47-fd01d304a9e9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   e3 5f 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ._..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.959689085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24086","req_id":"eeef1b2e-322f-48cb-af47-fd01d304a9e9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 58207,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 58207,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.960176543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24086","req_id":"eeef1b2e-322f-48cb-af47-fd01d304a9e9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.960389576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:55.960450277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:55.960491158Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":18,"elapsed":"6.180094ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.587696459Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.590549816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.590635507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":18,"elapsed":"47.755µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.591768707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:14:56.591839338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.591872463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":18,"elapsed":"52.775µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.591909196Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.591941951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.59198145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.59202198Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.592060266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.592629266Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.594399602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":18,"elapsed":"6.73662ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.595420209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.599025059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.599110659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.599152743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.599498139Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.601790481Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":18,"elapsed":"9.935202ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.723880021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.724011372Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.72411122Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.724187262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.724255347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.724757905Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.728618963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.731861393Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.731942815Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.731983516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.732294253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:56.739058357Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.739160179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":18,"elapsed":"15.333394ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.771194792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:14:56.771280913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.771313318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":18,"elapsed":"134.247µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.772451188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.772540896Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:56.773117992Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51255","req_id":"af8afe7f-f93f-41bb-a039-695e047e5ca8","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   7f af 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.773198823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51255","req_id":"af8afe7f-f93f-41bb-a039-695e047e5ca8","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32687,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32687,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:56.773634147Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51255","req_id":"af8afe7f-f93f-41bb-a039-695e047e5ca8","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:14:56.773809205Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:56.773868352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":18,"elapsed":"1.435561ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.724292924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":34,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.724484314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.72453737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.724582058Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.724620174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.725264844Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.728732131Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.733502104Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.733694005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.73377162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.733841548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.73680799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":34,"elapsed":"12.535697ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.924273842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":34,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.924404111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.924458189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.924509841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.924556093Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.925120584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.928595916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.932740478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:14:57.932828813Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.932884304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.933254088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:14:57.93604463Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":34,"elapsed":"11.748293ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.059968859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":88,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.060076733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.060129528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.060178575Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.060221851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.060834989Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.064371212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.069079202Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.069165574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.069207417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.069564436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.071899774Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":24,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:00.071957319Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.071990144Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.072032157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.072079Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.072119952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.072498684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.073076381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.073190368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":88,"elapsed":"13.1666ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.075424094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.079498897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.079585779Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.079632532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.079677361Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.079715978Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.080005563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.082970933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.086984273Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.087059513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.087102618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.087582502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:00.146200001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.146280501Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:00.146998106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:63316"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.148039184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"e40b617b-6ead-49cc-b0a9-43cf3383b643","remote_addr":"[::1]:63316","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.148176967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"e40b617b-6ead-49cc-b0a9-43cf3383b643","remote_addr":"[::1]:63316","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:00.148629486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"e40b617b-6ead-49cc-b0a9-43cf3383b643","remote_addr":"[::1]:63316","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":719,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.1493517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e40b617b-6ead-49cc-b0a9-43cf3383b643\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:15:00 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.149429946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.149475787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.150347518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:00.172100351Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.172233966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:00.172764751Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"9867a6a6-66da-4c11-9cfa-061aab415339","remote_addr":"[::1]:63316","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.17283976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"9867a6a6-66da-4c11-9cfa-061aab415339","remote_addr":"[::1]:63316","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:00.17333826Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"9867a6a6-66da-4c11-9cfa-061aab415339","remote_addr":"[::1]:63316","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":588,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.173922039Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"9867a6a6-66da-4c11-9cfa-061aab415339\", \"date\": \"Wed, 04 Mar 2026 07:15:00 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:00.173993731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":24,"elapsed":"102.014058ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.658619588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":32,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.658752723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.658798173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.65967768Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.661469829Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.661533726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.661574397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.661610789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.662132114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.665396187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.667795522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.670831412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.670891581Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.671347848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.672813689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.672872556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.672912435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.672948376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.673778565Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.677041365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.679841557Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:04.685183937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:04.685282563Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:04.685320408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:04.68535126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":32,"elapsed":"26.746751ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.059689125Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":89,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.059777209Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.05982262Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.059865765Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.059904723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.060640785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.064637382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.070329126Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.070455838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.07050232Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.070887556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.073997743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.0740772Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":89,"elapsed":"14.37505ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.306520482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62009","req_id":"94b52edb-48b3-4e55-a748-78a750fcb6b1","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   3c bd 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   <............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.306769447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62009","req_id":"94b52edb-48b3-4e55-a748-78a750fcb6b1","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 15549,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 15549,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:05.30743099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:62009","req_id":"94b52edb-48b3-4e55-a748-78a750fcb6b1","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.307615046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50241","req_id":"e07bb152-d160-425b-b60d-b2718b339499","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   89 7a 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .z..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.30777842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50241","req_id":"e07bb152-d160-425b-b60d-b2718b339499","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 35194,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 35194,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:05.308473802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50241","req_id":"e07bb152-d160-425b-b60d-b2718b339499","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:05.368632051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49427","req_id":"ed4774cc-119c-41d1-a00c-5b77c4cf443a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   3c fd 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   <............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:05.368756237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49427","req_id":"ed4774cc-119c-41d1-a00c-5b77c4cf443a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 15613,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 15613,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:05.369192965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49427","req_id":"ed4774cc-119c-41d1-a00c-5b77c4cf443a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:06.993959454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:06.99421925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:06.994983318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:60672"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:06.995506928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6cc6f4ad-8924-48c4-8f3d-0254bbc47b76","remote_addr":"127.0.0.1:60672","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:06.995577408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6cc6f4ad-8924-48c4-8f3d-0254bbc47b76","remote_addr":"127.0.0.1:60672","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:06.996117741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:07.002131615Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"6cc6f4ad-8924-48c4-8f3d-0254bbc47b76","remote_addr":"127.0.0.1:60672","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":6616,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:07.002509606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6cc6f4ad-8924-48c4-8f3d-0254bbc47b76\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:15:07 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 36.99148153s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:07.002584625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6cc6f4ad-8924-48c4-8f3d-0254bbc47b76\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:15:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6cc6f4ad-8924-48c4-8f3d-0254bbc47b76\" }"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.060203067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":90,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.060341722Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.060392423Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.06043651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.060477151Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.061576094Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.064981056Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.069923643Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.069998852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.070041787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.070443605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.073465427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.073542269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":90,"elapsed":"13.385375ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.402096996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":17,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.402186684Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.402234159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.403090049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.415958837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.416090468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.416772172Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.428755422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.428863587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.42891041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.428949447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.429448679Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.434236949Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.437735828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.437809484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.437852149Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.437894443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.437932378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.438217034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.440942617Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.444218794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.444290907Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.444329814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.444694288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.499669293Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:10.500135248Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:65375"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:10.500706914Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"2f2190f2-d2e7-4d83-8033-cca1a418315f","remote_addr":"[::1]:65375","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.500781121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"2f2190f2-d2e7-4d83-8033-cca1a418315f","remote_addr":"[::1]:65375","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:10.500938133Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"2f2190f2-d2e7-4d83-8033-cca1a418315f","remote_addr":"[::1]:65375","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":264,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.501244562Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2f2190f2-d2e7-4d83-8033-cca1a418315f\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:15:10 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:10.501318398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:10.501355211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:10.50138958Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:10.50142483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":17,"elapsed":"99.387492ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.723562579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":35,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.723686725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.72373427Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.723777516Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.723816764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.72450972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.727885535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.733266311Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.733353114Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.733394536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.733797707Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.736678348Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":35,"elapsed":"13.155237ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.924037318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":35,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.924161023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.924216213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.924267986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.92431513Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.925109177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.928866379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.936501002Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:12.936692652Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.93678726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.937483373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:12.941064225Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":35,"elapsed":"17.032418ms"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:14.865916941Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:41494"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:14.866160915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"38240925-cee8-4a81-959a-85ff25a0a5e3","remote_addr":"127.0.0.1:41494","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:14.86622367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"38240925-cee8-4a81-959a-85ff25a0a5e3","remote_addr":"127.0.0.1:41494","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:14.866460861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"38240925-cee8-4a81-959a-85ff25a0a5e3","remote_addr":"127.0.0.1:41494","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:14.866539708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"38240925-cee8-4a81-959a-85ff25a0a5e3","remote_addr":"127.0.0.1:41494","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:14.867199167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"38240925-cee8-4a81-959a-85ff25a0a5e3","remote_addr":"127.0.0.1:41494","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:14.870762334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"38240925-cee8-4a81-959a-85ff25a0a5e3","remote_addr":"127.0.0.1:41494","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":4596,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.060129783Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":91,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.060217407Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.06026397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.060309661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.060351444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.061159398Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.065344731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.070515931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.070621441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.070665538Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.071092787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.074278334Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.074388613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":91,"elapsed":"14.228669ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.310948009Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58994","req_id":"d58a0392-f8e4-49f5-93a9-7e0196d2483f","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   f1 b4 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.311054781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58994","req_id":"d58a0392-f8e4-49f5-93a9-7e0196d2483f","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 61876,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 61876,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:15.311485397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58994","req_id":"d58a0392-f8e4-49f5-93a9-7e0196d2483f","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.311529204Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50941","req_id":"35ff21e6-35ec-4474-abac-167b65afdf39","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   c3 3b 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   .;..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.311579874Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50941","req_id":"35ff21e6-35ec-4474-abac-167b65afdf39","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 49979,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 49979,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:15.311835923Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:50941","req_id":"35ff21e6-35ec-4474-abac-167b65afdf39","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:15.370887975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58319","req_id":"51c79424-4865-42e3-b43c-702d4c636f3e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   b5 0c 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:15.371163602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58319","req_id":"51c79424-4865-42e3-b43c-702d4c636f3e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 46348,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 46348,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:15.371616291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58319","req_id":"51c79424-4865-42e3-b43c-702d4c636f3e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.981249848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.981374556Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"internal","background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:17.981428082Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.981485777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.982236969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:17.984220869Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:17.984310607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:17.984375796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.984459142Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.984794328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.987759256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.994399795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:17.996070243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:17.996183658Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0\"}","iteration":10,"elapsed":"14.897678ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.056547946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:18.056643486Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.056686752Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":10,"elapsed":"105.961µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.577667954Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.577793904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_nameservice._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.578442903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63970","req_id":"0218804a-b80c-4c88-89ab-a5004e272392","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 75 (0x4b) bytes\n0000:   b8 d9 01 00  00 01 00 00  00 00 00 01  0c 5f 6e 61   ............._na\n0010:   6d 65 73 65  72 76 69 63  65 04 5f 74  63 70 0d 63   meservice._tcp.c\n0020:   6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65  05 6f 78 69   ontrol-plane.oxi\n0030:   64 65 08 69  6e 74 65 72  6e 61 6c 00  00 21 00 01   de.internal..!..\n0040:   00 00 29 04  d0 00 00 00  00 00 00                   ..)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.578519946Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63970","req_id":"0218804a-b80c-4c88-89ab-a5004e272392","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 47321,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 47321,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            12,\n            95,\n            110,\n            97,\n            109,\n            101,\n            115,\n            101,\n            114,\n            118,\n            105,\n            99,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_nameservice._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:18.579013787Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:63970","req_id":"0218804a-b80c-4c88-89ab-a5004e272392","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:18.579165037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_nameservice._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.57931761Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_internal","dns_group":"internal","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_nameservice._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":10,"elapsed":"1.639857ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.579381808Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"reading DNS version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.579419883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"dns_group":"external","background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.579456777Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.579502387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.580476602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.582773734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.58284725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.582895246Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.582935846Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.583637289Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.586856653Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.593242186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read DNS config","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:18.594814017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected exactly one latest version for DNS group External, found 0"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.594942933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_config_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0\"}","iteration":10,"elapsed":"15.313524ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.625951168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.626221475Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_external-dns._tcp.control-plane.oxide.internal"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.626460881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"DNS propagation: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:18.626556551Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no config nor servers"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.626600788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_propagation_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no config nor servers\"}","iteration":10,"elapsed":"73.476µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.626642471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"Metric producer GC running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:18.626675517Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","expiration":"2026-03-04 07:05:18.626555028 UTC"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.626707641Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.626746588Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.627361389Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:18.628337067Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:48133","req_id":"369b34b0-a40d-4da5-9122-205606a98f23","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 76 (0x4c) bytes\n0000:   26 a5 01 00  00 01 00 00  00 00 00 01  0d 5f 65 78   &............_ex\n0010:   74 65 72 6e  61 6c 2d 64  6e 73 04 5f  74 63 70 0d   ternal-dns._tcp.\n0020:   63 6f 6e 74  72 6f 6c 2d  70 6c 61 6e  65 05 6f 78   control-plane.ox\n0030:   69 64 65 08  69 6e 74 65  72 6e 61 6c  00 00 21 00   ide.internal..!.\n0040:   01 00 00 29  04 d0 00 00  00 00 00 00                ...)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.628420322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:48133","req_id":"369b34b0-a40d-4da5-9122-205606a98f23","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 9893,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 9893,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            13,\n            95,\n            101,\n            120,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            45,\n            100,\n            110,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_external-dns._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:18.628798473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:48133","req_id":"369b34b0-a40d-4da5-9122-205606a98f23","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to lookup DNS servers","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:18.628938211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_external-dns._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.629027047Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"dns_servers_external","dns_group":"external","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read list of DNS servers: proto error: no records found for Query { name: Name(\\\"_external-dns._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":10,"elapsed":"3.063083ms"}
{"msg":"Metric producer GC complete (no errors)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:18.630487237Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned":"{}","expiration":"2026-03-04 07:05:18.626555028 UTC"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:18.630548559Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"metrics_producer_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"expiration\":\"2026-03-04T07:05:18.626555028Z\",\"pruned\":[]}","iteration":10,"elapsed":"3.935366ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.264741178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.264902458Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.264957868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.265001595Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.265056674Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.265757256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.269139253Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.273424514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.27352855Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.273577397Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.274085336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.27789903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.278043708Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.27809542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.278135099Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.278606816Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.282572903Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.288417681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"DnsConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.288524544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.288565976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.288935059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"failed to read Silo/DNS/TLS configuration","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:19.292202338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Internal Error: expected at least one external DNS zone"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.292269371Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"external_endpoints","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone\"}","iteration":10,"elapsed":"27.57162ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.323457834Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"Decommissioned Disk Cleaner disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.323589025Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.323627792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"decommissioned_disk_cleaner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"deleted\":0,\"error\":null,\"error_count\":0,\"found\":0,\"not_ready_to_be_deleted\":0}","iteration":10,"elapsed":"80.3µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.45579688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.455873633Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.455917279Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.456608552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Found abandoned VMMs","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.460246077Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","total":0,"count":0}
{"msg":"Abandoned VMMs reaped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.460315885Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","vmms_already_deleted":0,"vmms_deleted":0,"sled_reservations_deleted":0,"vmms_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.460356195Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"abandoned_vmm_reaper","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"sled_reservations_deleted\":0,\"vmms_already_deleted\":0,\"vmms_deleted\":0,\"vmms_found\":0}","iteration":10,"elapsed":"4.549766ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.497453594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.497545256Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.497589975Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.498259354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.501323179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"lookup_region_port","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"found_port_ok\":[]}","iteration":10,"elapsed":"3.886499ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.5305417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.530633502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.530676447Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.531485374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reprocessing 0 outdated webhook globs...","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.53654385Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.536743226Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.536806352Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.537489789Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"webhook dispatching completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.540645336Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","events_without_receivers":0,"events_dispatched":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.540736347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"alert_dispatcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"dispatched\":[],\"errors\":[],\"glob_version\":\"234.0.0\",\"globs_reprocessed\":{},\"no_receivers\":[]}","iteration":10,"elapsed":"10.221581ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.645101274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.645187836Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.645231042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.646103956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all webhook delivery tasks completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.649652243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","num_receivers":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.649719477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"webhook_deliverator","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"by_rx\":{},\"error\":null}","iteration":10,"elapsed":"4.63735ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.658627449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":33,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.658707908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.658753439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.659410624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.661015692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.661086212Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.661131111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.661170199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.661550163Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.664461235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.666868085Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.669682665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.669751161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.670222417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.671547419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"multicast group reconciler activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.671612628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"starting multicast reconciliation pass","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.671645143Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"skipping cache invalidation check: no inventory available","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.671678208Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.671708268Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.672074606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.672140737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.67218243Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.672219053Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.672650239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.67355192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.673616859Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.673659263Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.673699884Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.673739322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.674205478Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.675412134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30283","req_id":"50eea14e-4327-4502-9775-b39493e0dbe4","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   7f cf 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.675491241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30283","req_id":"50eea14e-4327-4502-9775-b39493e0dbe4","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 32719,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 32719,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.67587812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:30283","req_id":"50eea14e-4327-4502-9775-b39493e0dbe4","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.676126443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to build DPD clients","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.676187304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"multicast RPW reconciliation pass completed with dataplane inconsistencies","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.676227123Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dataplane_error_count":1,"orphaned_members_cleaned":0,"member_state_transitions":0,"active_groups_verified":0,"external_groups_deleted":0,"external_groups_created":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.67636135Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"multicast_reconciler","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"empty_groups_marked\":0,\"errors\":[\"failed to create multicast dataplane client: Internal Error: failed to build DPD clients\"],\"groups_created\":0,\"groups_deleted\":0,\"groups_verified\":0,\"members_deleted\":0,\"members_processed\":0}","iteration":10,"elapsed":"4.739242ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.676464194Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.678782739Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.67905006Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.682365304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.682496845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.682546654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.682831851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.683745786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"starting attached subnet manager work","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.683798581Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.68382875Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.684360877Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:40459","req_id":"44969af5-e3b7-45bc-aadd-832fc88861b9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   74 bf 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   t............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.684439363Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:40459","req_id":"44969af5-e3b7-45bc-aadd-832fc88861b9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 29887,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 29887,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.684738337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:40459","req_id":"44969af5-e3b7-45bc-aadd-832fc88861b9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.685021199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to lookup Dendrite clients, will not be able to forward attachments to Dendrite","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:19.685078452Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.685119153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.685157469Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.68565637Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"Loaded 0 active trust quorum configurations from database","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.686377362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"bgtask":"TrustQuorumManager","bgtask":"TrustQuorumManager","background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.686430848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"trust_quorum_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"PerRackStatus\":{\"errors\":[],\"statuses\":[]}}","iteration":10,"elapsed":"12.920782ms"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.691021305Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.691099821Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.691132186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.691219018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":33,"elapsed":"32.592261ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.695513317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":10,"reason":"Timeout"}
{"msg":"distributing networking probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.69557042Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.69560049Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.695636532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.695676681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.695711731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.696428976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.699463152Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.703385301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.703466974Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.703509979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.703552233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.703664215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.703900344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"listed attached subnets","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.706185543Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_subnets":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.706250001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"attached_subnet_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"db_error\":null,\"dendrite\":{},\"sled\":{}}","iteration":10,"elapsed":"22.522131ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.706985301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.710973823Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.711051597Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.711097659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.711613533Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.715550773Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.715708616Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.715798555Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.715870157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.71649081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.719552892Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.72352456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.723588426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.723628797Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.724121615Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.785813089Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/probes","method":"PUT"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.786526065Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:51359"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.787479138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"f72c145d-3291-4960-aa4c-1e675b946f2c","remote_addr":"[::1]:51359","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.787571933Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"f72c145d-3291-4960-aa4c-1e675b946f2c","remote_addr":"[::1]:51359","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.787944012Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/probes","method":"PUT","req_id":"f72c145d-3291-4960-aa4c-1e675b946f2c","remote_addr":"[::1]:51359","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":497,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.788505908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/probes\", status: 204, headers: {\"x-request-id\": \"f72c145d-3291-4960-aa4c-1e675b946f2c\", \"date\": \"Wed, 04 Mar 2026 07:15:19 GMT\"} })"}
{"msg":"Update set of probes on sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.788572851Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_probes":0,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"finished distributing probes to sleds","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.788750824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","n_pushed_probes":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.78887968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"probe_distributor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"probes_by_sled\":{\"b6d65341-167c-41df-9b5c-41cded99c229\":0}}","iteration":10,"elapsed":"93.369179ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.789131981Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":25,"reason":"Dependency"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.789185628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.789216749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.78925721Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.789298051Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.789337349Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.790588895Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.794267741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.799569521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.799677115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.799721853Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.799765129Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.799804598Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.800176106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.804390676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.812419402Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.812518819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.812561213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.81317379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.902722381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.902911156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.903362602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:45583"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.904096931Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"fdfad66c-fbe4-42f8-b02f-074fdecd7628","remote_addr":"[::1]:45583","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.904200567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"fdfad66c-fbe4-42f8-b02f-074fdecd7628","remote_addr":"[::1]:45583","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.904245366Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"fdfad66c-fbe4-42f8-b02f-074fdecd7628","remote_addr":"[::1]:45583","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":241,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.90475087Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fdfad66c-fbe4-42f8-b02f-074fdecd7628\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:15:19 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.904838764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.905037359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.905886215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.920282276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.920442374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:19.921538161Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"e2fb4cff-3930-44e4-bd21-8c9e2a2c56d3","remote_addr":"[::1]:45583","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.921624111Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"e2fb4cff-3930-44e4-bd21-8c9e2a2c56d3","remote_addr":"[::1]:45583","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:19.921881523Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"e2fb4cff-3930-44e4-bd21-8c9e2a2c56d3","remote_addr":"[::1]:45583","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":446,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.922550792Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"e2fb4cff-3930-44e4-bd21-8c9e2a2c56d3\", \"date\": \"Wed, 04 Mar 2026 07:15:19 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:19.922854665Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":25,"elapsed":"133.625982ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.059894438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":92,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:20.059973344Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:20.060018153Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:20.060080968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.06012213Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.060825968Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.064112174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.069867786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:20.069945059Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.069989327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.070687544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.073543115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:20.073764795Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":92,"elapsed":"13.900247ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.059983064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":93,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.060096689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.060139374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.060178041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.060214493Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.060981276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.064640594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.069536297Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.069616737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.069658299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.070022483Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.073145185Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.073217799Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":93,"elapsed":"13.247771ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.313523897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65134","req_id":"73bac0b5-ddeb-44ef-a18c-dba435e1e602","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   d3 ff 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.313670007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65134","req_id":"73bac0b5-ddeb-44ef-a18c-dba435e1e602","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 54271,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 54271,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.314017327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65134","req_id":"73bac0b5-ddeb-44ef-a18c-dba435e1e602","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.314061705Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56697","req_id":"85b3fc4c-16c1-413b-955e-e6f323da7987","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   6f 3a 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   o:..........._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.314113628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56697","req_id":"85b3fc4c-16c1-413b-955e-e6f323da7987","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 28474,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 28474,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.314387482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:56697","req_id":"85b3fc4c-16c1-413b-955e-e6f323da7987","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.373473552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59442","req_id":"d2224d1b-96f9-4386-b6d7-bd37522c74c1","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   64 4c 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   dL..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.373607267Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59442","req_id":"d2224d1b-96f9-4386-b6d7-bd37522c74c1","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 25676,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 25676,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.374012422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:59442","req_id":"d2224d1b-96f9-4386-b6d7-bd37522c74c1","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.624678536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.624802362Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.624848663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.626092233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.629274804Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.630311663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:38343","req_id":"d5e875f2-bdc3-4c75-a8d0-0424192999a0","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   c9 ba 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.630451241Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:38343","req_id":"d5e875f2-bdc3-4c75-a8d0-0424192999a0","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 51642,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 51642,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.631062605Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:38343","req_id":"d5e875f2-bdc3-4c75-a8d0-0424192999a0","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.631310928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.631372771Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.63155342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":19,"elapsed":"6.927869ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.954103681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.954235542Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.954281504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.954802338Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.958125408Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:25.958923613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:16298","req_id":"55c0bfdf-94df-4045-9c53-84950392f95a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   3c d1 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   <............_de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.959007239Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:16298","req_id":"55c0bfdf-94df-4045-9c53-84950392f95a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 15569,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 15569,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.959418576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:16298","req_id":"55c0bfdf-94df-4045-9c53-84950392f95a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.959580157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:25.959638944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:25.959676108Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":19,"elapsed":"5.616064ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.588333894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.591063445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.5911663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":19,"elapsed":"58.316µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.591448951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:26.591517888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.591550913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":19,"elapsed":"117.634µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.591588378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.591623688Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.591664249Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.591706893Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.591747003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.592436412Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.593174298Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":19,"elapsed":"4.897236ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.595508624Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.598998746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.59907672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.599121068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.599423529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.602218971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":19,"elapsed":"10.644992ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.724589429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.724735419Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.724787001Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.724830167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.724866599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.72544081Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.729558909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.733231252Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.733320299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.733361541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.733696456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:26.741212434Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.741291561Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":19,"elapsed":"16.815026ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.770501504Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:26.770595321Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.770629238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":19,"elapsed":"165.168µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.771772819Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.771826766Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:26.772282531Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:27430","req_id":"cc84b21e-fe72-450a-ad19-0ea8c011eb99","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   65 e0 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   e............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.772354745Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:27430","req_id":"cc84b21e-fe72-450a-ad19-0ea8c011eb99","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26080,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26080,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:26.772798396Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:27430","req_id":"cc84b21e-fe72-450a-ad19-0ea8c011eb99","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:26.772975868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:26.773078472Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":19,"elapsed":"1.291796ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.724148235Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":36,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.724219146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.724262502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.724305187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.724345167Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.725012602Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.72835965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.733147178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.733211676Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.733253018Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.733609105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.736163749Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":36,"elapsed":"12.024812ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.923794359Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":36,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.9239259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.923980489Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.92403171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.924096219Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.924731651Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.928174318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.932312417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:27.932412064Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.932463016Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.932821307Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:27.935499857Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":36,"elapsed":"11.75687ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.05992449Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":94,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.060039358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.060087524Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.060133074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.060173284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.060882422Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.064200021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.069070424Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.069148179Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.06919381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.069529406Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.071926807Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":26,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:30.071990203Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.072025283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.072066244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.072108238Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.072147345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.072518944Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.073574169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.073641303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":94,"elapsed":"13.793725ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.075134578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.079684124Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.07975775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.079802529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.079843841Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.079883691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.080232714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.083876781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.088084438Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.088157303Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.088200068Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.088600102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:30.14142184Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.141537741Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:30.142079988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:35890"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.143005015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"fdb4f598-f3e3-49b2-8a96-d13368597196","remote_addr":"[::1]:35890","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.143094873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"fdb4f598-f3e3-49b2-8a96-d13368597196","remote_addr":"[::1]:35890","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:30.14363713Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"fdb4f598-f3e3-49b2-8a96-d13368597196","remote_addr":"[::1]:35890","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":668,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.144301971Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fdb4f598-f3e3-49b2-8a96-d13368597196\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:15:30 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.144360908Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.144404083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.145235785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:30.165608277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.16570037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:30.166230442Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"14fbcec0-a2d4-43cd-8690-df1ebe4ad9e9","remote_addr":"[::1]:35890","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.166322866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"14fbcec0-a2d4-43cd-8690-df1ebe4ad9e9","remote_addr":"[::1]:35890","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:30.166715586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"14fbcec0-a2d4-43cd-8690-df1ebe4ad9e9","remote_addr":"[::1]:35890","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":514,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.167132514Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"14fbcec0-a2d4-43cd-8690-df1ebe4ad9e9\", \"date\": \"Wed, 04 Mar 2026 07:15:30 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:30.167297592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":26,"elapsed":"95.362608ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.658637803Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":34,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.658753162Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.658803192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.659531578Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.661177988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.661247005Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.661289379Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.661325982Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.661678482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.664706827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.667163817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.670064468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.670153304Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.670920568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.672810642Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.672889488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.672937604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.672977283Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.673496564Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.676605008Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.679165383Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:34.685691785Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:34.685813456Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:34.685850961Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:34.685881301Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":34,"elapsed":"27.235342ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.059990453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":95,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.060072335Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.06012011Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.06016525Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.060207003Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.061236257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.064242337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.069520921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.069616731Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.069659476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.070032587Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.073184216Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.073263544Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":95,"elapsed":"13.270216ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.316973382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54257","req_id":"75718575-df35-4397-9d86-c30e8a435d77","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   90 5f 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ._..........._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.317076116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54257","req_id":"75718575-df35-4397-9d86-c30e8a435d77","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 36959,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 36959,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:35.317778471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54257","req_id":"75718575-df35-4397-9d86-c30e8a435d77","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.317824522Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60656","req_id":"57fdfc6f-14c0-4065-bc14-fbd493086419","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   f5 c9 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   ............._re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.317874842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60656","req_id":"57fdfc6f-14c0-4065-bc14-fbd493086419","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 62921,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 62921,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:35.31813038Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60656","req_id":"57fdfc6f-14c0-4065-bc14-fbd493086419","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:35.375275244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49441","req_id":"24199d55-4696-4012-ac44-f03911b4814a","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   b2 bf 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:35.375383109Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49441","req_id":"24199d55-4696-4012-ac44-f03911b4814a","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 45759,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 45759,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:35.375871388Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49441","req_id":"24199d55-4696-4012-ac44-f03911b4814a","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.060353827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":96,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.060466711Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.060515097Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.060559155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.060599034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.061279034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.064854966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.069811951Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.069915567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.069961759Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.070292937Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.073187115Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.073269328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":96,"elapsed":"12.887465ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.402308888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":18,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.402447333Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.40252155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.403410997Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.418436909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.418536567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.419292889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.428841714Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.428943055Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.428992484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.429032663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.429682193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.432512675Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.436092214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.436171071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.436216712Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.436258825Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.436297953Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.436533661Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.438789201Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.442011781Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.442124435Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.442171188Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.442483207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.499449337Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:40.500134898Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:53903"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:40.501378007Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"73f0eeb6-8382-49bd-a323-5bf27019a90b","remote_addr":"[::1]:53903","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.501448467Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"73f0eeb6-8382-49bd-a323-5bf27019a90b","remote_addr":"[::1]:53903","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:40.50165612Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"73f0eeb6-8382-49bd-a323-5bf27019a90b","remote_addr":"[::1]:53903","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":316,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.501999802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73f0eeb6-8382-49bd-a323-5bf27019a90b\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:15:40 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:40.502069681Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:40.502105071Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:40.502137145Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:40.502168607Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":18,"elapsed":"99.910491ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.724255585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":37,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.724402897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.724473037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.724534358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.724593576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.725170191Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.729394852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.734620701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.734803764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.734876439Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.735087618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.73785299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":37,"elapsed":"13.77594ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.924218277Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":37,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.924372413Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.924440178Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.924498403Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.924559845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.925522507Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.930169136Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.935284325Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:42.935420425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.935482258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.935802033Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:42.938367659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":37,"elapsed":"14.206395ms"}
{"msg":"contacting server nexus","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:43.995638095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:43.99585723Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","body":"Some(Body)","uri":"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:43.996459076Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:42734"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:43.996857186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"37f41dab-836b-4500-a8da-196bf635eeb2","remote_addr":"127.0.0.1:42734","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:43.99693443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"37f41dab-836b-4500-a8da-196bf635eeb2","remote_addr":"127.0.0.1:42734","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"registered sled agent","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:43.997345837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","sled_uuid":"b6d65341-167c-41df-9b5c-41cded99c229"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:44.003325453Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229","method":"POST","req_id":"37f41dab-836b-4500-a8da-196bf635eeb2","remote_addr":"127.0.0.1:42734","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","error_message_external":"Internal Server Error","error_message_internal":"Unknown diesel error creating Sled called \"b6d65341-167c-41df-9b5c-41cded99c229\": Record not found","latency_us":6460,"response_code":500}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:44.003694015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"NexusClient","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","result":"Ok(Response { url: \"http://127.0.0.1:52506/sled-agents/b6d65341-167c-41df-9b5c-41cded99c229\", status: 500, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"37f41dab-836b-4500-a8da-196bf635eeb2\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:15:44 GMT\"} })"}
{"msg":"failed to contact nexus, will retry in 37.157922168s","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:44.003850526Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"37f41dab-836b-4500-a8da-196bf635eeb2\", \"content-length\": \"124\", \"date\": \"Wed, 04 Mar 2026 07:15:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"37f41dab-836b-4500-a8da-196bf635eeb2\" }"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:44.905491288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","remote_addr":"127.0.0.1:49247"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:44.905752166Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d15bed06-089e-443e-a83b-d68465d63746","remote_addr":"127.0.0.1:49247","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:44.905820622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d15bed06-089e-443e-a83b-d68465d63746","remote_addr":"127.0.0.1:49247","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","version":"1.0.0"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:44.906079887Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d15bed06-089e-443e-a83b-d68465d63746","remote_addr":"127.0.0.1:49247","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:44.906166159Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d15bed06-089e-443e-a83b-d68465d63746","remote_addr":"127.0.0.1:49247","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:44.906940737Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"user_builtin_id":"001de000-05e4-4000-8000-000000000002","type":"user_builtin","authenticated":true,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d15bed06-089e-443e-a83b-d68465d63746","remote_addr":"127.0.0.1:49247","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:44.911225848Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/nat/ipv4/changeset/0?limit=100","method":"GET","req_id":"d15bed06-089e-443e-a83b-d68465d63746","remote_addr":"127.0.0.1:49247","local_addr":"127.0.0.1:52506","component":"dropshot_internal","name":"913233fe-92a8-4635-9572-183f495429c4","latency_us":5439,"response_code":200}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.059681132Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":97,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.059818645Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.05986625Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.059920257Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.059964274Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.06052604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.064276529Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.069186791Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.069285798Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.069330186Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.069621865Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.072437046Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.07251414Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":97,"elapsed":"12.876294ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.319957613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65021","req_id":"7c74287a-ed17-4a79-8820-516d9398d408","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   71 8a 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   q............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.320116037Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65021","req_id":"7c74287a-ed17-4a79-8820-516d9398d408","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 29066,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 29066,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:45.320500762Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:65021","req_id":"7c74287a-ed17-4a79-8820-516d9398d408","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.320763223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49221","req_id":"7871bb7f-2393-49dc-9ab1-bde4e1c584e9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   4b 80 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   K............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.320834484Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49221","req_id":"7871bb7f-2393-49dc-9ab1-bde4e1c584e9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 19328,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 19328,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:45.321106965Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:49221","req_id":"7871bb7f-2393-49dc-9ab1-bde4e1c584e9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:45.378930728Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54270","req_id":"ecbc52ec-278c-4a20-8965-65b0bf7d5637","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   81 06 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   ............._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:45.379031839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54270","req_id":"ecbc52ec-278c-4a20-8965-65b0bf7d5637","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 33030,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 33030,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:45.379521221Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:54270","req_id":"ecbc52ec-278c-4a20-8965-65b0bf7d5637","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.659349211Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":35,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.659447627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.659490632Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.660321322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.662381784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.662448236Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.662489979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.662524828Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.663065482Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.666868315Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.669306888Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.672141328Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.672214192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.672729536Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.674325375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.674392288Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.674436376Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.674471796Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.674907331Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.678059031Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.680377976Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:49.687886549Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:49.68795738Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:49.687991157Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:49.688019964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":35,"elapsed":"28.698337ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.0603692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":98,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:50.060485541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:50.060568015Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:50.060643906Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.060708925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.061193327Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.065012532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.069993465Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:50.070070347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.070113814Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.070497285Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.073264091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:50.073347967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":98,"elapsed":"12.989247ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.059588461Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":99,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.059731966Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.059780192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.059821784Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.059862866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.060604629Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.064578972Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.06944156Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.069556057Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.069605365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.069883488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.072329956Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.072406889Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":99,"elapsed":"12.861023ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.323063374Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58417","req_id":"a435078e-edac-419d-a3d2-16f40cd58264","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   a3 c1 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.323238742Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58417","req_id":"a435078e-edac-419d-a3d2-16f40cd58264","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 41921,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 41921,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.323699437Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:58417","req_id":"a435078e-edac-419d-a3d2-16f40cd58264","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.32376654Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57553","req_id":"202f3eb7-9ead-4c6c-b760-f553f741597c","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   21 a0 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   !............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.323859014Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57553","req_id":"202f3eb7-9ead-4c6c-b760-f553f741597c","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 8608,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 8608,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.324327715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57553","req_id":"202f3eb7-9ead-4c6c-b760-f553f741597c","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.38189592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60020","req_id":"95803212-7f14-4819-ac76-f4fae1ff862e","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   e4 52 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   .R..........._cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.382033744Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60020","req_id":"95803212-7f14-4819-ac76-f4fae1ff862e","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 58450,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 58450,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.3824345Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:60020","req_id":"95803212-7f14-4819-ac76-f4fae1ff862e","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.623842065Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.623932024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.623977464Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.624721782Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.627527354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.628559655Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42912","req_id":"c7930e26-9211-47b2-a701-55071626df8d","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   07 cd 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   ............._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.628673471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42912","req_id":"c7930e26-9211-47b2-a701-55071626df8d","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 1997,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 1997,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.629155488Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:42912","req_id":"c7930e26-9211-47b2-a701-55071626df8d","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.629463339Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.629528569Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.629566925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"nat_garbage_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":20,"elapsed":"5.781042ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.954075817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.954169382Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.954211426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.954851207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"lookup_all_ipv6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.958034048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_dendrite._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:55.959021088Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24051","req_id":"c29ff9d3-c574-48ff-b7cb-53fe9ee82445","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 72 (0x48) bytes\n0000:   71 5d 01 00  00 01 00 00  00 00 00 01  09 5f 64 65   q]..........._de\n0010:   6e 64 72 69  74 65 04 5f  74 63 70 0d  63 6f 6e 74   ndrite._tcp.cont\n0020:   72 6f 6c 2d  70 6c 61 6e  65 05 6f 78  69 64 65 08   rol-plane.oxide.\n0030:   69 6e 74 65  72 6e 61 6c  00 00 21 00  01 00 00 29   internal..!....)\n0040:   04 d0 00 00  00 00 00 00                             ........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.959261436Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24051","req_id":"c29ff9d3-c574-48ff-b7cb-53fe9ee82445","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 29021,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 29021,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            9,\n            95,\n            100,\n            101,\n            110,\n            100,\n            114,\n            105,\n            116,\n            101,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_dendrite._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.959580169Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:24051","req_id":"c29ff9d3-c574-48ff-b7cb-53fe9ee82445","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.959941576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"failed to resolve addresses for Dendrite services","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:55.96000898Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"proto error: no records found for Query { name: Name(\"_dendrite._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:55.960051044Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"bfd_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name(\\\"_dendrite._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":20,"elapsed":"5.94662ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.588712518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.59082837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.590895894Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"physical_disk_adoption","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"task disabled\"}","iteration":20,"elapsed":"81.653µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.592311566Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"inventory collection is None","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:15:56.592378258Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.592438999Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"service_zone_nat_tracker","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"inventory collection is None\"}","iteration":20,"elapsed":"66.782µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.592478146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.592511663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.592551963Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.59259586Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.59263591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.593369296Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.594342649Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"phantom_disks","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"phantom_disk_deleted_err\":0,\"phantom_disk_deleted_ok\":0}","iteration":20,"elapsed":"5.721193ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.596388322Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.599965347Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.600041017Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.600084343Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.600464378Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.603313276Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"switch_port_config_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":20,"elapsed":"10.889177ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.723844411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.723972074Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.724019699Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.724063837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.724102473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.724734969Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.728047368Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.731599693Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.731677568Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.731725764Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.732032102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"all instance checks complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:56.738880043Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","pruned_instances":0,"update_sagas_queued":0,"total_incomplete":0,"total_failed":0,"total_completed":0,"total_instances":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.738978138Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"failed_checks\":{},\"incomplete_checks\":{},\"instance_states\":{},\"pruned_instances\":0,\"total_instances\":0,\"update_sagas_queued\":0}","iteration":20,"elapsed":"15.185981ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.77115842Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"background instance updater explicitly disabled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:15:56.771302146Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.771340802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_updater","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"destroyed_active_vmms\":0,\"disabled\":true,\"failed_active_vmms\":0,\"query_errors\":[],\"saga_errors\":[],\"sagas_completed\":0,\"sagas_started\":0,\"terminated_active_migrations\":0}","iteration":20,"elapsed":"238.714µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.771458426Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":20,"reason":"Timeout"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.771738843Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:56.772168476Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:44961","req_id":"73edbf8c-627a-40af-94f0-be24d39eb9a2","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   e1 cf 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.772259788Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:44961","req_id":"73edbf8c-627a-40af-94f0-be24d39eb9a2","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 57807,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 57807,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:56.772784881Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:44961","req_id":"73edbf8c-627a-40af-94f0-be24d39eb9a2","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"failed to resolve MGS addresses","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:15:56.772932223Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"Resolve(ResolveError { kind: Proto(ProtoError { kind: NoRecordsFound { query: Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, soa: None, ns: None, negative_ttl: None, response_code: ServFail, trusted: true, authorities: None } }) })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:56.772994938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"sp_ereport_ingester","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"disabled\":false,\"errors\":[\"failed to resolve MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"],\"sps\":[]}","iteration":20,"elapsed":"1.607112ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.723789023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":38,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.723894964Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.723945574Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.723991365Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.724053468Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.724877775Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.728385171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.733861177Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.733987518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.734034532Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.734394326Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.737339806Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":38,"elapsed":"13.537968ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.924450913Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":38,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.924601733Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.92466062Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.924716521Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.924768604Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.925197286Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.92861317Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.932768904Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:15:57.932897118Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.932952618Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.933201192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:15:57.935544897Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":38,"elapsed":"11.146668ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.060259495Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":100,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.060347299Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.060396567Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.060440314Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.06047852Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.061239942Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.064988477Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.06991443Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.069996173Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.070041663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.07036746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.0718628Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":27,"reason":"Timeout"}
{"msg":"VPC route manager running","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:00.071932107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.071966967Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.072010112Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.07207421Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.072115872Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.072441189Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.0752919Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.075371098Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":100,"elapsed":"15.106342ms"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.075481627Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.078851019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.078924214Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.078967921Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.079007991Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.079044724Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.079330411Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.081936417Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.085441909Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.085509664Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.085550986Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.085954537Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"VPC route manager sled b6d65341-167c-41df-9b5c-41cded99c229","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:00.143498925Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.143619354Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/vpc-routes","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:00.144176591Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:41372"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.145610278Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"a7851ddf-4c14-4ab8-82be-5ed2f18419ad","remote_addr":"[::1]:41372","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.145674746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"a7851ddf-4c14-4ab8-82be-5ed2f18419ad","remote_addr":"[::1]:41372","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:00.147253171Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/vpc-routes","method":"GET","req_id":"a7851ddf-4c14-4ab8-82be-5ed2f18419ad","remote_addr":"[::1]:41372","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":1652,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.148563584Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/vpc-routes\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a7851ddf-4c14-4ab8-82be-5ed2f18419ad\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:16:00 GMT\"} })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.148750866Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.148811576Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.150205515Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"computed internet gateway mappings for sled","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:00.170915827Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","assocs":"{}","sled":"SimGimlet00"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.171011748Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"Some(Body)","uri":"http://[::1]:33621/eip-gateways","method":"PUT"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:00.171627952Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"c1893d71-00bc-4e74-b40f-e1bed08e3e57","remote_addr":"[::1]:41372","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.171893168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"c1893d71-00bc-4e74-b40f-e1bed08e3e57","remote_addr":"[::1]:41372","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:00.175513839Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/eip-gateways","method":"PUT","req_id":"c1893d71-00bc-4e74-b40f-e1bed08e3e57","remote_addr":"[::1]:41372","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":3912,"response_code":204}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.17646571Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/eip-gateways\", status: 204, headers: {\"x-request-id\": \"c1893d71-00bc-4e74-b40f-e1bed08e3e57\", \"date\": \"Wed, 04 Mar 2026 07:16:00 GMT\"} })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:00.176540518Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"vpc_route_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":27,"elapsed":"104.69841ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.658819224Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":36,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.658929302Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.658974883Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.659820091Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.661698592Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.661778812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.661824222Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.661862979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.662346519Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.665556725Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.668043734Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.670914907Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.670984174Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.675016312Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.677205631Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.677278726Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.677324116Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.677363644Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.677877715Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.680908926Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.683431746Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"deleted 0 of 0 sitreps sitreps","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:04.689770636Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","case_ereports_deleted":0,"cases_deleted":0,"sitreps_deleted":0,"ids":"[]"}
{"msg":"all 0 orphaned sitreps in this batch have already been deleted","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:04.689912678Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"sitrep garbage collection found 0 orphaned sitreps, deleted 0","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:04.68994916Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:04.689980613Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_gc","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"errors\":[],\"orphaned_sitreps_deleted\":0,\"orphaned_sitreps_found\":0}","iteration":36,"elapsed":"31.193112ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.060406358Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":101,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.060543511Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.060590955Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.060633019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.060672868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.061298019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.064585689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.069734244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.069810506Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.069850215Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.070292794Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.073277401Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.073350957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":101,"elapsed":"12.997084ms"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.326291769Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64740","req_id":"a01550ac-bbe6-4b8a-a725-ee4356bf8aff","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   17 ee 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   ............._mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.326434272Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64740","req_id":"a01550ac-bbe6-4b8a-a725-ee4356bf8aff","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 6126,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 6126,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:16:05.326801351Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:64740","req_id":"a01550ac-bbe6-4b8a-a725-ee4356bf8aff","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.326843444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57243","req_id":"580e9abb-1859-4da1-9cf3-64f6db780a65","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 74 (0x4a) bytes\n0000:   23 b5 01 00  00 01 00 00  00 00 00 01  0b 5f 72 65   #............_re\n0010:   70 6f 2d 64  65 70 6f 74  04 5f 74 63  70 0d 63 6f   po-depot._tcp.co\n0020:   6e 74 72 6f  6c 2d 70 6c  61 6e 65 05  6f 78 69 64   ntrol-plane.oxid\n0030:   65 08 69 6e  74 65 72 6e  61 6c 00 00  21 00 01 00   e.internal..!...\n0040:   00 29 04 d0  00 00 00 00  00 00                      .)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.32689638Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57243","req_id":"580e9abb-1859-4da1-9cf3-64f6db780a65","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 9141,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 9141,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_repo-depot._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            11,\n            95,\n            114,\n            101,\n            112,\n            111,\n            45,\n            100,\n            101,\n            112,\n            111,\n            116,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_repo-depot._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:16:05.327168871Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:57243","req_id":"580e9abb-1859-4da1-9cf3-64f6db780a65","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:05.384153066Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51554","req_id":"a4bca8f2-b5c3-4eb0-b183-8981f321db58","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 79 (0x4f) bytes\n0000:   6e cb 01 00  00 01 00 00  00 00 00 01  10 5f 63 72   n............_cr\n0010:   75 63 69 62  6c 65 2d 70  61 6e 74 72  79 04 5f 74   ucible-pantry._t\n0020:   63 70 0d 63  6f 6e 74 72  6f 6c 2d 70  6c 61 6e 65   cp.control-plane\n0030:   05 6f 78 69  64 65 08 69  6e 74 65 72  6e 61 6c 00   .oxide.internal.\n0040:   00 21 00 01  00 00 29 04  d0 00 00 00  00 00 00      .!....)........"}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:05.384264528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51554","req_id":"a4bca8f2-b5c3-4eb0-b183-8981f321db58","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 28363,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 28363,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_crucible-pantry._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            16,\n            95,\n            99,\n            114,\n            117,\n            99,\n            105,\n            98,\n            108,\n            101,\n            45,\n            112,\n            97,\n            110,\n            116,\n            114,\n            121,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_crucible-pantry._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:16:05.384728048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:51554","req_id":"a4bca8f2-b5c3-4eb0-b183-8981f321db58","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.060262805Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":102,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.060383284Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.060437021Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.060483373Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.060528743Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.061093915Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.065037868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.069904233Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"BlueprintConfig","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.0700034Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.070050113Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.070322083Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"reconfigurator config load complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.073775121Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","switches":"Loaded(ReconfiguratorConfigView { version: 1, config: ReconfiguratorConfig { planner_enabled: false, planner_config: PlannerConfig { add_zones_with_mupdate_override: false }, tuf_repo_pruner_enabled: true }, time_modified: 2026-03-04T07:06:13.816186Z })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.073887845Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"reconfigurator_config_watcher","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"config_updated\":false}","iteration":102,"elapsed":"13.602846ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.401937168Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":19,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.402104861Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.402155342Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.402881594Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.416132541Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.416228552Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.416908462Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.426585512Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.426725259Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.42677601Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.426817873Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.427448275Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.430208838Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.433663269Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.433736023Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.433781323Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.433823928Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.433864849Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.434135106Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.437346444Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.440685095Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.440760244Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.440806817Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.441154367Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"client request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.501273579Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","body":"None","uri":"http://[::1]:33621/v2p","method":"GET"}
{"msg":"accepted connection","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:10.501804473Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","remote_addr":"[::1]:35652"}
{"msg":"incoming request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:10.502589943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"6ad9140c-36f1-429e-aa97-b302a125a822","remote_addr":"[::1]:35652","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server"}
{"msg":"determined request API version","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.502718528Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"6ad9140c-36f1-429e-aa97-b302a125a822","remote_addr":"[::1]:35652","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","version":"24.0.0"}
{"msg":"request completed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:10.502766824Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"uri":"/v2p","method":"GET","req_id":"6ad9140c-36f1-429e-aa97-b302a125a822","remote_addr":"[::1]:35652","local_addr":"[::1]:33621","component":"dropshot","sled_id":"b6d65341-167c-41df-9b5c-41cded99c229","component":"omicron_sled_agent::sim::Server","latency_us":324,"response_code":200}
{"msg":"client response","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.50299105Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"SledAgent":"b6d65341-167c-41df-9b5c-41cded99c229","background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(Response { url: \"http://[::1]:33621/v2p\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ad9140c-36f1-429e-aa97-b302a125a822\", \"content-length\": \"2\", \"date\": \"Wed, 04 Mar 2026 07:16:10 GMT\"} })"}
{"msg":"found opte v2p mappings","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:10.503080868Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","interfaces":"{}","sled":"SimGimlet00"}
{"msg":"v2p mappings to delete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:10.503119535Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"v2p mappings to add","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:10.503151669Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","mappings":"[]","sled":"SimGimlet00"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:10.503188291Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"v2p_manager","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{}","iteration":19,"elapsed":"101.33526ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.724154192Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":39,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.724307577Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.724358548Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.72440451Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.724447024Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.725071695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.728635793Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.733581927Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Read","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.733712987Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.733756924Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.734066199Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.736632786Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoCollections\"","iteration":39,"elapsed":"12.563031ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.923983369Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":39,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.924125692Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.924184028Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.924236502Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.924284207Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.924811193Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.92829387Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.931803441Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Fleet","action":"ListChildren","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:12.931891445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.931944811Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.932232593Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:12.935006041Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"fm_sitrep_loader","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"NoSitrep\"","iteration":39,"elapsed":"11.106178ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.960853048Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_executor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"Blueprint execution: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:16:13.960948107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_executor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no blueprint"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.960988898Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_executor","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no blueprint\"}","iteration":2,"elapsed":"97.534µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.961048837Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.961113996Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Inventory","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"loading roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.961158695Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource_id":"001de000-1334-4000-8000-000000000000","resource_type":"Fleet","actor":"Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. }"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.961202672Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.961242622Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.961890509Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.965466701Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, \"admin\")} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.96956988Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Inventory","action":"Modify","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.969684107Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.969731802Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.970057479Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.97102454Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":3,"reason":"Timeout"}
{"msg":"blueprint planning disabled, doing nothing","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.971168155Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.971206812Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_planner","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"\"Disabled\"","iteration":3,"elapsed":"65.249µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.971244957Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"crdb_node_id_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"Blueprint execution: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:16:13.971280318Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"crdb_node_id_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no blueprint"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.971312962Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"crdb_node_id_collector","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no blueprint\"}","iteration":2,"elapsed":"21.984µs"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.971349445Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_rendezvous","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"Blueprint rendezvous: skipped","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:16:13.971383663Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_rendezvous","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reason":"no blueprint"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.971415606Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"blueprint_rendezvous","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"no blueprint\"}","iteration":2,"elapsed":"13.827µs"}
{"msg":"inventory_prune_one: nothing eligible for removal (too few)","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.97455425Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","candidates":"[]"}
{"msg":"lookup_all_socket_v6 srv","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.974620471Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"DnsResolver","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","dns_name":"_mgs._tcp.control-plane.oxide.internal"}
{"msg":"buffer","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.975439187Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26121","req_id":"f223c9c7-6987-47c9-90a6-d65fc1e127e9","component":"dns","kind":"dns","component":"internal_dns_server","buffer":"Length: 67 (0x43) bytes\n0000:   67 0a 01 00  00 01 00 00  00 00 00 01  04 5f 6d 67   g............_mg\n0010:   73 04 5f 74  63 70 0d 63  6f 6e 74 72  6f 6c 2d 70   s._tcp.control-p\n0020:   6c 61 6e 65  05 6f 78 69  64 65 08 69  6e 74 65 72   lane.oxide.inter\n0030:   6e 61 6c 00  00 21 00 01  00 00 29 04  d0 00 00 00   nal..!....).....\n0040:   00 00 00                                             ..."}
{"msg":"message_request","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.975513134Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26121","req_id":"f223c9c7-6987-47c9-90a6-d65fc1e127e9","component":"dns","kind":"dns","component":"internal_dns_server","mr":"MessageRequest {\n    header: Header {\n        id: 26378,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n}"}
{"msg":"failed to handle incoming DNS message: MessageRequest {\n    header: Header {\n        id: 26378,\n        message_type: Query,\n        op_code: Query,\n        authoritative: false,\n        truncation: false,\n        recursion_desired: true,\n        recursion_available: false,\n        authentic_data: false,\n        checking_disabled: false,\n        response_code: NoError,\n        query_count: 1,\n        answer_count: 0,\n        name_server_count: 0,\n        additional_count: 1,\n    },\n    queries: Queries {\n        queries: [\n            LowerQuery {\n                name: LowerName(\n                    Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                ),\n                original: Query {\n                    name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"),\n                    query_type: SRV,\n                    query_class: IN,\n                },\n            },\n        ],\n        original: [\n            4,\n            95,\n            109,\n            103,\n            115,\n            4,\n            95,\n            116,\n            99,\n            112,\n            13,\n            99,\n            111,\n            110,\n            116,\n            114,\n            111,\n            108,\n            45,\n            112,\n            108,\n            97,\n            110,\n            101,\n            5,\n            111,\n            120,\n            105,\n            100,\n            101,\n            8,\n            105,\n            110,\n            116,\n            101,\n            114,\n            110,\n            97,\n            108,\n            0,\n            0,\n            33,\n            0,\n            1,\n        ],\n    },\n    answers: [],\n    name_servers: [],\n    additionals: [],\n    sig0: [],\n    edns: Some(\n        Edns {\n            rcode_high: 0,\n            version: 0,\n            flags: EdnsFlags {\n                dnssec_ok: false,\n                z: 0,\n            },\n            max_payload: 1232,\n            options: OPT {\n                options: [],\n            },\n        },\n    ),\n} SERVFAIL: server is not authoritative for name: \"_mgs._tcp.control-plane.oxide.internal.\"","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:16:13.975929912Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"peer_addr":"[::1]:26121","req_id":"f223c9c7-6987-47c9-90a6-d65fc1e127e9","component":"dns","kind":"dns","component":"internal_dns_server"}
{"msg":"inventory collection failed","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":40,"time":"2026-03-04T07:16:13.97620585Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","error":"failed to collect inventory: looking up MGS addresses: proto error: no records found for Query { name: Name(\"_mgs._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.97631148Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"inventory_collection","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"error\":\"failed to collect inventory: looking up MGS addresses: proto error: no records found for Query { name: Name(\\\"_mgs._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"}","iteration":3,"elapsed":"15.270779ms"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.979543659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.979634019Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.979682355Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.980152689Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"activating","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.983761727Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"saga_recovery","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","iteration":2,"reason":"Timeout"}
{"msg":"saga recovery pass starting","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:13.983833659Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaQuiesceHandle","component":"NexusQuiesceHandle","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"listing candidate sagas for recovery","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.983871404Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4"}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:13.983900833Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000003a8a (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.983940381Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:13.984366979Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000003a8a (built_in_user), .. })"}
{"msg":"listed in-progress sagas","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:14.046660599Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaRecoverer","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","count":0}
{"msg":"saga recovery pass finished","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:14.046739255Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"SagaQuiesceHandle","component":"NexusQuiesceHandle","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","blueprint_id":"None","generation":"1"}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:14.046900375Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"saga_recovery","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"last_pass\":{\"Success\":{\"nfailed\":0,\"nfound\":0,\"nrecovered\":0,\"nremoved\":0,\"nskipped\":0}},\"ntotal_failures\":0,\"ntotal_finished\":0,\"ntotal_recovered\":0,\"ntotal_sec_errors_bad_state\":0,\"ntotal_sec_errors_missing\":0,\"ntotal_started\":0,\"recent_failures\":[],\"recent_recoveries\":[]}","iteration":2,"elapsed":"63.037568ms"}
{"msg":"no more instances in need of reincarnation","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:14.150768646Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reincarnation_reason":"instance failed","total_found":0}
{"msg":"authorize begin","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:14.150877943Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"roles","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:14.150925217Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","roles":"RoleSet { roles: {} }"}
{"msg":"authorize result","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:14.151982938Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","result":"Ok(())","resource":"Database","action":"Query","actor":"Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002 (built_in_user), .. })"}
{"msg":"no more instances in need of reincarnation","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":10,"time":"2026-03-04T07:16:14.337690218Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","reincarnation_reason":"start saga failed","total_found":0}
{"msg":"instance reincarnation completed successfully","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":30,"time":"2026-03-04T07:16:14.337791429Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","instances_changed_state":0,"instances_reincarnated":0,"instances_found":0}
{"msg":"activation complete","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":20,"time":"2026-03-04T07:16:14.337832691Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"background_task":"instance_reincarnation","component":"BackgroundTasks","component":"nexus","component":"ServerContext","name":"913233fe-92a8-4635-9572-183f495429c4","status":"{\"changed_state\":[],\"disabled\":false,\"errors\":[],\"instances_found\":{\"Failed\":0,\"SagaUnwound\":0},\"instances_reincarnated\":[],\"restart_errors\":[]}","iteration":2,"elapsed":"358.351486ms"}
{"msg":"Timed out after 600s while running step start_sled1, failing test","v":0,"name":"test_upstairs_notify_downstairs_client_stops","level":50,"time":"2026-03-04T07:16:14.584968102Z","hostname":"bmat-EVT22200007-0000eafd","pid":3471,"component":"init_with_steps"}