01JSDMAK6FDQPZAM8Q7PC1TNXM: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.34",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JSDMBJ2GCQWJ4D8WQT88F8SQ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-04-22T02:49:40.366Zjob dependencies complete; ready to run (waiting for 35 m 58 s)
22025-04-22T02:53:28.975Zjob assigned to worker 01JSDP8YJ3XCFGGAEBPFTCNH29 [factory lab, buskin/14616] (queued for 3 m 48 s)
32025-04-22T02:53:28.995Zhttp://172.20.0.5:9969/ipxe/buskin... ok
42025-04-22T02:53:29.050Zhttp://172.20.0.5:9969/os/buskin/platform/i86pc/kernel/amd64/unix... ok
52025-04-22T02:53:29.084Zhttp://172.20.0.5:9969/os/buskin/platform/i86pc/amd64/boot_archive... ok
62025-04-22T02:53:29.117Zhttp://172.20.0.5:9969/os/buskin/platform/i86pc/amd64/boot_archive.hash... ok
72025-04-22T02:53:29.150Zhttp://172.20.0.5:9969/postboot/buskin... ok
82025-04-22T02:53:29.184ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92025-04-22T02:53:29.217Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d5
102025-04-22T02:53:29.251ZOxide Helios Version helios-2.0.23304 64-bit
112025-04-22T02:53:29.295ZConfiguring devices.
122025-04-22T02:53:29.329Z##################################################
132025-04-22T02:53:29.362Z#### ############# ##
142025-04-22T02:53:29.395Z### ### ############ ##
152025-04-22T02:53:29.429Z## ### # ## ### ## Oxide Computer Company ##
162025-04-22T02:53:29.462Z## ## ## ### # ### ##
172025-04-22T02:53:29.496Z## # ### #### #### This Station Under ##
182025-04-22T02:53:29.529Z### ### #### # ### Computer Control ##
192025-04-22T02:53:29.563Z#### #### ### ## ##
202025-04-22T02:53:29.607Z##################################################
212025-04-22T02:53:29.641ZHostname: buskin
222025-04-22T02:53:29.674ZLoading smf(7) service descriptions:
232025-04-22T02:53:30.470Zdownloading input: /input/package/work/package.tar.gz
242025-04-22T02:59:20.778Zdownloaded input: /input/package/work/package.tar.gz
 
252025-04-22T02:59:20.803Zstarting task 0: "setup"
262025-04-22T02:59:20.803Z++ uname -s
272025-04-22T02:59:20.803Z+ kern=SunOS
282025-04-22T02:59:20.803Z+ build_user=build
292025-04-22T02:59:20.803Z+ build_uid=12345
302025-04-22T02:59:20.803Z+ work_dir=/work
312025-04-22T02:59:20.803Z+ input_dir=/input
322025-04-22T02:59:20.803Z+ [[ 0 == 12345 ]]
332025-04-22T02:59:20.803Z+ case "$kern" in
342025-04-22T02:59:20.803Z+ groupadd -g 12345 build
352025-04-22T02:59:20.804Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
362025-04-22T02:59:22.688Z+ zfs create -o mountpoint=/work rpool/work
372025-04-22T02:59:22.732Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
382025-04-22T02:59:22.756Z+ home_fs=zfs
392025-04-22T02:59:22.756Z+ [[ zfs == autofs ]]
402025-04-22T02:59:22.756Z+ mkdir -p /home/build
412025-04-22T02:59:22.756Z+ chown build:build /home/build /work
422025-04-22T02:59:24.739Z+ chmod 0700 /home/build /work
432025-04-22T02:59:24.764Zprocess exited: duration 4073 ms, exit code 0
 
442025-04-22T02:59:24.813Zstarting task 1: "authentication"
452025-04-22T02:59:24.860Zprocess exited: duration 46 ms, exit code 0
 
462025-04-22T02:59:24.911Zstarting task 2: "build"
472025-04-22T02:59:24.936Z+ trap _exit_trap EXIT
482025-04-22T02:59:24.936Z+ OPTE_COMMIT=
492025-04-22T02:59:24.936Z+ [[ x != \x ]]
502025-04-22T02:59:24.936Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
512025-04-22T02:59:25.028Zsecpolicy_fs_linkdir: 0x55 = 0x55
522025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
532025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
542025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+3: 0 = 0xe5
552025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+4: 0 = 0xb8
562025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+5: 0 = 0x1
572025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
582025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
592025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
602025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
612025-04-22T02:59:25.028Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
622025-04-22T02:59:25.054Z+ [[ -d /opt/oxide ]]
632025-04-22T02:59:25.054Z+ pfexec mv /opt/oxide /opt/oxide-underneath
642025-04-22T02:59:25.054Z+ pfexec mkdir /opt/oxide
652025-04-22T02:59:25.054Z+ pfexec mount -F tmpfs -O swap /opt/oxide
662025-04-22T02:59:25.054Z+ [[ -d /opt/oxide-underneath ]]
672025-04-22T02:59:25.054Z+ cd /opt/oxide-underneath
682025-04-22T02:59:25.054Z+ pfexec tar ceEp@/f - .
692025-04-22T02:59:25.054Z+ cd /opt/oxide
702025-04-22T02:59:25.054Z+ pfexec tar xveEp@/f -
712025-04-22T02:59:25.088Zx ., 0 bytes, 0 tape blocks
722025-04-22T02:59:25.088Zx ./opte, 0 bytes, 0 tape blocks
732025-04-22T02:59:25.088Zx ./opte/bin, 0 bytes, 0 tape blocks
742025-04-22T02:59:25.088Zx ./opte/bin/opteadm, 17540664 bytes, 34260 tape blocks
752025-04-22T02:59:25.112Z+ pfexec rm -rf /opt/oxide-underneath
762025-04-22T02:59:25.112Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
772025-04-22T02:59:25.112Z+ pfexec mkdir /opt/oxide/work
782025-04-22T02:59:25.138Z+ pfexec chown build:build /opt/oxide/work
792025-04-22T02:59:25.138Z+ cd /opt/oxide/work
802025-04-22T02:59:25.138Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
812025-04-22T02:59:40.148ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
822025-04-22T02:59:40.148Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
832025-04-22T02:59:40.148Zx out/target/test, 107 bytes, 1 tape blocks
842025-04-22T02:59:40.148Zx out/npuzone/npuzone, 23699288 bytes, 46288 tape blocks
852025-04-22T02:59:40.148Zx package-manifest.toml, 33785 bytes, 66 tape blocks
862025-04-22T02:59:40.148Zx smf/sled-agent/non-gimlet/config.toml, 4942 bytes, 10 tape blocks
872025-04-22T02:59:40.148Zx target/release/omicron-package, 41059112 bytes, 80194 tape blocks
882025-04-22T02:59:40.148Zx target/release/xtask, 11018904 bytes, 21522 tape blocks
892025-04-22T02:59:40.148Zx target/debug/bootstrap, 215284048 bytes, 420477 tape blocks
902025-04-22T02:59:40.148Zx tests/bootstrap-c6b0d27f0cb47d17, 24404608 bytes, 47666 tape blocks
912025-04-22T02:59:40.148Zx tests/commtest-38aab2992c80b05a, 25210592 bytes, 49240 tape blocks
922025-04-22T02:59:40.148Zx tests/dhcp_server-1c4aecca2e768282, 24404624 bytes, 47666 tape blocks
932025-04-22T02:59:40.148Zx tests/end_to_end_tests-804095052923f3b9, 272645712 bytes, 532512 tape blocks
942025-04-22T02:59:40.148Zx out/clickhouse.tar.gz, 312823159 bytes, 610983 tape blocks
952025-04-22T02:59:40.148Zx out/clickhouse_keeper.tar.gz, 297843234 bytes, 581726 tape blocks
962025-04-22T02:59:40.148Zx out/clickhouse_server.tar.gz, 312805441 bytes, 610949 tape blocks
972025-04-22T02:59:40.149Zx out/cockroachdb.tar.gz, 148315362 bytes, 289679 tape blocks
982025-04-22T02:59:40.149Zx out/crucible-dtrace.tar, 101376 bytes, 198 tape blocks
992025-04-22T02:59:40.149Zx out/crucible-pantry-zone.tar.gz, 34631706 bytes, 67641 tape blocks
1002025-04-22T02:59:40.149Zx out/crucible-zone.tar.gz, 43364741 bytes, 84697 tape blocks
1012025-04-22T02:59:40.149Zx out/external-dns.tar.gz, 46921058 bytes, 91643 tape blocks
1022025-04-22T02:59:40.149Zx out/internal-dns.tar.gz, 46921214 bytes, 91643 tape blocks
1032025-04-22T02:59:40.149Zx out/mg-ddm-gz.tar, 67185664 bytes, 131222 tape blocks
1042025-04-22T02:59:40.149Zx out/nexus.tar.gz, 115227366 bytes, 225054 tape blocks
1052025-04-22T02:59:40.149Zx out/ntp.tar.gz, 15339387 bytes, 29960 tape blocks
1062025-04-22T02:59:40.149Zx out/omicron-gateway-softnpu.tar.gz, 25797186 bytes, 50386 tape blocks
1072025-04-22T02:59:40.149Zx out/omicron-sled-agent.tar, 127866368 bytes, 249739 tape blocks
1082025-04-22T02:59:40.149Zx out/overlay.tar.gz, 2622 bytes, 6 tape blocks
1092025-04-22T02:59:40.149Zx out/oximeter.tar.gz, 56871938 bytes, 111079 tape blocks
1102025-04-22T02:59:40.149Zx out/oxlog.tar, 3850752 bytes, 7521 tape blocks
1112025-04-22T02:59:40.149Zx out/probe.tar.gz, 2829470 bytes, 5527 tape blocks
1122025-04-22T02:59:40.149Zx out/propolis-server.tar.gz, 70529499 bytes, 137753 tape blocks
1132025-04-22T02:59:40.149Zx out/pumpkind-gz.tar, 27287040 bytes, 53295 tape blocks
1142025-04-22T02:59:40.149Zx out/switch-softnpu.tar.gz, 283139352 bytes, 553007 tape blocks
1152025-04-22T02:59:40.175Z
1162025-04-22T02:59:40.175Zreal 15.005437290
1172025-04-22T02:59:40.175Zuser 13.165237924
1182025-04-22T02:59:40.175Zsys 3.096601073
1192025-04-22T02:59:40.175Ztrap 0.000649404
1202025-04-22T02:59:40.175Ztflt 0.000000000
1212025-04-22T02:59:40.175Zdflt 0.000000000
1222025-04-22T02:59:40.175Zkflt 0.000000000
1232025-04-22T02:59:40.175Zlock 0.000000000
1242025-04-22T02:59:40.175Zslp 13.308847778
1252025-04-22T02:59:40.175Zlat 0.437219023
1262025-04-22T02:59:40.175Zstop 0.000129103
1272025-04-22T02:59:40.175Z+ source .github/buildomat/ci-env.sh
1282025-04-22T02:59:40.175Z++ export CARGO_TERM_COLOR=always
1292025-04-22T02:59:40.175Z++ CARGO_TERM_COLOR=always
1302025-04-22T02:59:40.175Z+ bmat address ls
1312025-04-22T02:59:40.175ZNAME CIDR FIRST LAST COUNT
1322025-04-22T02:59:40.175Zextra 10.151.1.0/24 10.151.1.164 10.151.1.227 64
1332025-04-22T02:59:40.175Z++ bmat address ls -f extra -Ho first,last,count
1342025-04-22T02:59:40.175Z+ set -- 10.151.1.164 10.151.1.227 64
1352025-04-22T02:59:40.175Z+ EXTRA_IP_START=10.151.1.164
1362025-04-22T02:59:40.175Z+ EXTRA_IP_END=10.151.1.227
1372025-04-22T02:59:40.175Z+ EXTRA_IP_COUNT=64
1382025-04-22T02:59:40.175Z+ (( EXTRA_IP_COUNT >= 32 ))
1392025-04-22T02:59:40.175Z+ EXTRA_IP_BASE=10.151.1
1402025-04-22T02:59:40.176Z+ EXTRA_IP_FOCTET=164
1412025-04-22T02:59:40.176Z+ EXTRA_IP_LOCTET=227
1422025-04-22T02:59:40.176Z+ SERVICE_IP_POOL_START=10.151.1.164
1432025-04-22T02:59:40.176Z+ SERVICE_IP_POOL_END=10.151.1.173
1442025-04-22T02:59:40.176Z+ DNS_IP1=10.151.1.164
1452025-04-22T02:59:40.176Z+ DNS_IP2=10.151.1.165
1462025-04-22T02:59:40.176Z+ UPLINK_IP=10.151.1.174
1472025-04-22T02:59:40.176Z+ IPPOOL_START=10.151.1.175
1482025-04-22T02:59:40.176Z+ IPPOOL_END=10.151.1.227
1492025-04-22T02:59:40.176Z++ ipadm show-addr -po type,addr
1502025-04-22T02:59:40.176Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1512025-04-22T02:59:40.176Z+ GATEWAY_IP=10.151.1.100
1522025-04-22T02:59:40.176Z+ [[ -n 10.151.1.100 ]]
1532025-04-22T02:59:40.176Z+ ping -s 10.151.1.100 56 1
1542025-04-22T02:59:40.176ZPING 10.151.1.100: 56 data bytes
1552025-04-22T02:59:40.176Z64 bytes from 10.151.1.100: icmp_seq=0. time=0.012 ms
1562025-04-22T02:59:40.176Z
1572025-04-22T02:59:40.176Z----10.151.1.100 PING Statistics----
1582025-04-22T02:59:40.176Z1 packets transmitted, 1 packets received, 0% packet loss
1592025-04-22T02:59:40.176Zround-trip (ms) min/avg/max/stddev = 0.012/0.012/0.012/-nan
1602025-04-22T02:59:40.176Z++ arp -an
1612025-04-22T02:59:40.176Z++ awk -vgw=10.151.1.100 '$2 == gw {print $NF}'
1622025-04-22T02:59:40.201Z+ GATEWAY_MAC=18:c0:4d:0d:9f:b2
1632025-04-22T02:59:40.201Z+ routeadm -e ipv4-forwarding -u
1642025-04-22T02:59:40.290Z+ PXA_START=10.151.1.164
1652025-04-22T02:59:40.290Z+ PXA_END=10.151.1.227
1662025-04-22T02:59:40.290Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
1672025-04-22T02:59:40.447Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.1.100 --gateway-mac 18:c0:4d:0d:9f:b2 --pxa-start 10.151.1.164 --pxa-end 10.151.1.227
1682025-04-22T02:59:40.447ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1692025-04-22T02:59:40.507Zcreating virtual hardware
1702025-04-22T02:59:40.532Zcreating /scratch/m2_0.vdev
1712025-04-22T02:59:40.532Zcreating /scratch/m2_1.vdev
1722025-04-22T02:59:40.532Zcreating /scratch/u2_0.vdev
1732025-04-22T02:59:40.532Zcreating /scratch/u2_1.vdev
1742025-04-22T02:59:40.532Zcreating /scratch/u2_2.vdev
1752025-04-22T02:59:40.532Zcreating /scratch/u2_3.vdev
1762025-04-22T02:59:40.532Zcreating /scratch/u2_4.vdev
1772025-04-22T02:59:40.532Zcreating /scratch/u2_5.vdev
1782025-04-22T02:59:40.532Zcreating /scratch/u2_6.vdev
1792025-04-22T02:59:40.532Zcreating /scratch/u2_7.vdev
1802025-04-22T02:59:40.532Zcreating /scratch/u2_8.vdev
1812025-04-22T02:59:40.557ZSimnet net0/sc0_0 exists
1822025-04-22T02:59:40.582ZSimnet net1/sc1_0 exists
1832025-04-22T02:59:40.582ZVnic sc0_1 exists
1842025-04-22T02:59:53.124ZUsing 10.151.1.100 as gateway ip
1852025-04-22T02:59:53.124Zusing 18:C0:4D:0D:9F:B2 as gateway mac
1862025-04-22T02:59:53.124Zconfiguring SoftNPU ARP entry
1872025-04-22T02:59:53.171Zconfiguring SoftNPU proxy ARP
1882025-04-22T02:59:53.172ZSoftNPU state:
1892025-04-22T02:59:53.172Z local v6:
1902025-04-22T02:59:53.172Z local v4:
1912025-04-22T02:59:53.172Z router v6:
1922025-04-22T02:59:53.172Z router v4_idx:
1932025-04-22T02:59:53.172Z router v4_routes:
1942025-04-22T02:59:53.172Z resolver v4:
1952025-04-22T02:59:53.172Z 10.151.1.100 -> 18:c0:4d:0d:9f:b2
1962025-04-22T02:59:53.172Z resolver v6:
1972025-04-22T02:59:53.172Z nat v4:
1982025-04-22T02:59:53.172Z nat_v6:
1992025-04-22T02:59:53.172Z port_mac:
2002025-04-22T02:59:53.172Z proxy arp:
2012025-04-22T02:59:53.172Z 10.151.1.164/10.151.1.227: a8:e1:de:01:70:1d
2022025-04-22T02:59:53.172Zcreated virtual hardware
2032025-04-22T02:59:53.172Z
2042025-04-22T02:59:53.172Zreal 12.658505528
2052025-04-22T02:59:53.172Zuser 0.654872778
2062025-04-22T02:59:53.172Zsys 3.198196858
2072025-04-22T02:59:53.172Ztrap 0.000076077
2082025-04-22T02:59:53.172Ztflt 0.000209609
2092025-04-22T02:59:53.172Zdflt 0.000435863
2102025-04-22T02:59:53.172Zkflt 0.000005080
2112025-04-22T02:59:53.172Zlock 9:47.313602962
2122025-04-22T02:59:53.172Zslp 38.202634314
2132025-04-22T02:59:53.172Zlat 0.094976802
2142025-04-22T02:59:53.172Zstop 0.371001982
2152025-04-22T02:59:53.172Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2162025-04-22T02:59:53.172Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2172025-04-22T02:59:53.172Z+ diff -u pkg/config.toml~ pkg/config.toml
2182025-04-22T02:59:53.172Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2192025-04-22T02:59:53.172Z+++ pkg/config.toml Tue Apr 22 02:59:52 2025
2202025-04-22T02:59:53.172Z@@ -29,18 +29,18 @@
2212025-04-22T02:59:53.172Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2222025-04-22T02:59:53.172Z # followed by a string that is embedded into their fake serial values.
2232025-04-22T02:59:53.172Z vdevs = [
2242025-04-22T02:59:53.173Z- "m2_0.vdev",
2252025-04-22T02:59:53.173Z- "m2_1.vdev",
2262025-04-22T02:59:53.173Z+ "/scratch/m2_0.vdev",
2272025-04-22T02:59:53.173Z+ "/scratch/m2_1.vdev",
2282025-04-22T02:59:53.173Z
2292025-04-22T02:59:53.173Z- "u2_0.vdev",
2302025-04-22T02:59:53.173Z- "u2_1.vdev",
2312025-04-22T02:59:53.173Z- "u2_2.vdev",
2322025-04-22T02:59:53.173Z- "u2_3.vdev",
2332025-04-22T02:59:53.173Z- "u2_4.vdev",
2342025-04-22T02:59:53.173Z- "u2_5.vdev",
2352025-04-22T02:59:53.173Z- "u2_6.vdev",
2362025-04-22T02:59:53.173Z- "u2_7.vdev",
2372025-04-22T02:59:53.173Z- "u2_8.vdev",
2382025-04-22T02:59:53.173Z+ "/scratch/u2_0.vdev",
2392025-04-22T02:59:53.173Z+ "/scratch/u2_1.vdev",
2402025-04-22T02:59:53.173Z+ "/scratch/u2_2.vdev",
2412025-04-22T02:59:53.173Z+ "/scratch/u2_3.vdev",
2422025-04-22T02:59:53.173Z+ "/scratch/u2_4.vdev",
2432025-04-22T02:59:53.173Z+ "/scratch/u2_5.vdev",
2442025-04-22T02:59:53.173Z+ "/scratch/u2_6.vdev",
2452025-04-22T02:59:53.173Z+ "/scratch/u2_7.vdev",
2462025-04-22T02:59:53.173Z+ "/scratch/u2_8.vdev",
2472025-04-22T02:59:53.173Z ]
2482025-04-22T02:59:53.173Z
2492025-04-22T02:59:53.173Z # The amount of memory held back for services which exist between zero and one
2502025-04-22T02:59:53.173Z+ true
2512025-04-22T02:59:53.173Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2522025-04-22T02:59:53.173Z+ EXPECTED_ZPOOL_COUNT=9
2532025-04-22T02:59:53.173Z+ echo 'expected number of zpools is 9'
2542025-04-22T02:59:53.173Zexpected number of zpools is 9
2552025-04-22T02:59:53.173Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2562025-04-22T02:59:53.173Z+ SILO_NAME=recovery
2572025-04-22T02:59:53.173Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2582025-04-22T02:59:53.173Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2592025-04-22T02:59:53.173Z+ sed -i~ '
2602025-04-22T02:59:53.173Z /^external_dns_ips/c\
2612025-04-22T02:59:53.173Zexternal_dns_ips = [ "10.151.1.164", "10.151.1.165" ]
2622025-04-22T02:59:53.173Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2632025-04-22T02:59:53.173Z /^first/c\
2642025-04-22T02:59:53.173Zfirst = "10.151.1.164"
2652025-04-22T02:59:53.174Z /^last/c\
2662025-04-22T02:59:53.174Zlast = "10.151.1.173"
2672025-04-22T02:59:53.174Z }
2682025-04-22T02:59:53.174Z /^infra_ip_first/c\
2692025-04-22T02:59:53.174Zinfra_ip_first = "10.151.1.174"
2702025-04-22T02:59:53.174Z /^infra_ip_last/c\
2712025-04-22T02:59:53.174Zinfra_ip_last = "10.151.1.174"
2722025-04-22T02:59:53.174Z /^\[\[rack_network_config.ports/,/^$/ {
2732025-04-22T02:59:53.174Z /^routes/c\
2742025-04-22T02:59:53.174Zroutes = \[{nexthop = "10.151.1.100", destination = "0.0.0.0/0"}\]
2752025-04-22T02:59:53.174Z /^addresses/c\
2762025-04-22T02:59:53.174Zaddresses = \[{address = "10.151.1.174/24"} \]
2772025-04-22T02:59:53.174Z }
2782025-04-22T02:59:53.174Z' pkg/config-rss.toml
2792025-04-22T02:59:53.174Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2802025-04-22T02:59:53.199Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2812025-04-22T02:59:53.199Z+++ pkg/config-rss.toml Tue Apr 22 02:59:52 2025
2822025-04-22T02:59:53.199Z@@ -33,7 +33,7 @@
2832025-04-22T02:59:53.199Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2842025-04-22T02:59:53.199Z # must be contained in one of the "internal services" IP Pool ranges listed
2852025-04-22T02:59:53.199Z # below.
2862025-04-22T02:59:53.199Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2872025-04-22T02:59:53.199Z+external_dns_ips = [ "10.151.1.164", "10.151.1.165" ]
2882025-04-22T02:59:53.199Z
2892025-04-22T02:59:53.199Z # Initial TLS certificates for the external API
2902025-04-22T02:59:53.199Z #
2912025-04-22T02:59:53.199Z@@ -69,8 +69,8 @@
2922025-04-22T02:59:53.200Z #
2932025-04-22T02:59:53.200Z # For more on this and what to put here, see docs/how-to-run.adoc.
2942025-04-22T02:59:53.200Z [[internal_services_ip_pool_ranges]]
2952025-04-22T02:59:53.200Z-first = "192.168.1.20"
2962025-04-22T02:59:53.200Z-last = "192.168.1.29"
2972025-04-22T02:59:53.200Z+first = "10.151.1.164"
2982025-04-22T02:59:53.200Z+last = "10.151.1.173"
2992025-04-22T02:59:53.200Z
3002025-04-22T02:59:53.200Z # TODO - this configuration is subject to change going forward. Ultimately these
3012025-04-22T02:59:53.200Z # parameters should be provided to the control plane via wicket, but we need to
3022025-04-22T02:59:53.200Z@@ -91,8 +91,8 @@
3032025-04-22T02:59:53.200Z # A range of IP addresses used by Boundary Services on the external network. In
3042025-04-22T02:59:53.200Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3052025-04-22T02:59:53.200Z # With softnpu, only one address is used.
3062025-04-22T02:59:53.200Z-infra_ip_first = "192.168.1.30"
3072025-04-22T02:59:53.200Z-infra_ip_last = "192.168.1.30"
3082025-04-22T02:59:53.200Z+infra_ip_first = "10.151.1.174"
3092025-04-22T02:59:53.200Z+infra_ip_last = "10.151.1.174"
3102025-04-22T02:59:53.200Z
3112025-04-22T02:59:53.200Z # Configurations for BGP routers to run on the scrimlets.
3122025-04-22T02:59:53.200Z bgp = []
3132025-04-22T02:59:53.200Z@@ -100,9 +100,9 @@
3142025-04-22T02:59:53.200Z # You can configure multiple uplinks by repeating the following stanza
3152025-04-22T02:59:53.200Z [[rack_network_config.ports]]
3162025-04-22T02:59:53.200Z # Routes associated with this port.
3172025-04-22T02:59:53.200Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3182025-04-22T02:59:53.200Z+routes = [{nexthop = "10.151.1.100", destination = "0.0.0.0/0"}]
3192025-04-22T02:59:53.200Z # Addresses associated with this port.
3202025-04-22T02:59:53.200Z-addresses = [{address = "192.168.1.30/24"}]
3212025-04-22T02:59:53.200Z+addresses = [{address = "10.151.1.174/24"} ]
3222025-04-22T02:59:53.200Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3232025-04-22T02:59:53.200Z port = "qsfp0"
3242025-04-22T02:59:53.200Z # The speed of this port.
3252025-04-22T02:59:53.200Z+ true
3262025-04-22T02:59:53.200Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3272025-04-22T02:59:53.200Za pkg/config-rss.toml 8K
3282025-04-22T02:59:53.200Za pkg/config.toml 5K
3292025-04-22T02:59:53.200Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3302025-04-22T02:59:53.200Z+ TLS_NAME=recovery.sys.oxide.test
3312025-04-22T02:59:53.201Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3322025-04-22T02:59:53.354Z.........+.......+.....+.......+.....+.........+.......+..+.......+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+......+........+..........+...+..+.......+...+......+............+.....+....+......+..+...+..........+.....+.......+.....+..................+.......+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.............+....+..+.............+.....+............+...+.....................+.+........+.+...............+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3332025-04-22T02:59:53.784Z.+.............+.....+..........+..+................+..+.+..+...+.+......+.....+...+.+.....+....+.....+......+.+..+...+..................+....+...+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*......+...........+............+............+..........+........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+.+...+......+..............+...+...+.......+............+........+.........+.+.........+.....................+..+..........+............+.....+...+.+............+............+...+........+.........+................+..+.............+.........+..+.+...+...........+.+.........+.....+...+.............+...+..+.......+......+...........+......+.......+........+............+.......+......+.....+....+.....+......+........................+...............+.........+.........+...............+............+.+..+...+...+......+...+...+....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3342025-04-22T02:59:53.809Z-----
3352025-04-22T02:59:53.809Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3362025-04-22T02:59:53.834Za pkg/initial-tls-cert.pem 2K
3372025-04-22T02:59:53.834Za pkg/initial-tls-key.pem 4K
3382025-04-22T02:59:53.834Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3392025-04-22T02:59:53.834Z+ rmdir pkg
3402025-04-22T02:59:53.834Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3412025-04-22T02:59:53.834Z+ pfexec mkdir -p /usr/oxide
3422025-04-22T02:59:53.834Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v0.1.1
3432025-04-22T02:59:54.382Z+ pfexec chmod +x /usr/oxide/oxide
3442025-04-22T02:59:54.408Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3452025-04-22T03:00:12.697Z+ OMICRON_NO_UNINSTALL=1
3462025-04-22T03:00:12.697Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3472025-04-22T03:00:12.721ZLogging to: /opt/oxide/work/out/LOG
3482025-04-22T03:00:13.421Z
3492025-04-22T03:00:13.445Zreal 0.719182396
3502025-04-22T03:00:13.445Zuser 0.143516610
3512025-04-22T03:00:13.445Zsys 3.926899090
3522025-04-22T03:00:13.445Ztrap 0.000003880
3532025-04-22T03:00:13.446Ztflt 0.000005209
3542025-04-22T03:00:13.446Zdflt 0.000949159
3552025-04-22T03:00:13.446Zkflt 0.000000000
3562025-04-22T03:00:13.446Zlock 1:04.819906018
3572025-04-22T03:00:13.446Zslp 0.921548528
3582025-04-22T03:00:13.446Zlat 0.011423759
3592025-04-22T03:00:13.446Zstop 0.236780823
3602025-04-22T03:00:13.446Z+ retry=0
3612025-04-22T03:00:13.446Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3622025-04-22T03:00:13.446Z+ [[ 0 -gt 30 ]]
3632025-04-22T03:00:13.446Z+ sleep 1
3642025-04-22T03:00:14.436Z+ retry=1
3652025-04-22T03:00:14.436Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3662025-04-22T03:00:14.460Z+ [[ 1 -gt 30 ]]
3672025-04-22T03:00:14.461Z+ sleep 1
3682025-04-22T03:00:15.451Z+ retry=2
3692025-04-22T03:00:15.451Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3702025-04-22T03:00:15.476Z+ [[ 2 -gt 30 ]]
3712025-04-22T03:00:15.476Z+ sleep 1
3722025-04-22T03:00:16.466Z+ retry=3
3732025-04-22T03:00:16.466Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3742025-04-22T03:00:37.111Z+ [[ 3 -gt 30 ]]
3752025-04-22T03:00:37.111Z+ sleep 1
3762025-04-22T03:00:38.135Z+ retry=4
3772025-04-22T03:00:38.135Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3782025-04-22T03:00:38.161Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
3792025-04-22T03:00:38.202Zlocal v6:
3802025-04-22T03:00:38.202Zfe80::ac9b:65ff:fe57:f62a
3812025-04-22T03:00:38.202Zfdb8:9c64:5caa:edf3::1
3822025-04-22T03:00:38.202Zlocal v4:
3832025-04-22T03:00:38.202Z10.151.1.174
3842025-04-22T03:00:38.202Zrouter v6:
3852025-04-22T03:00:38.202Zfd00:1122:3344:2::/64 -> fe80::a430:eff:fe86:b0ce (1)
3862025-04-22T03:00:38.202Zrouter v4_idx:
3872025-04-22T03:00:38.202Z0.0.0.0/0 -> 0
3882025-04-22T03:00:38.202Zrouter v4_routes:
3892025-04-22T03:00:38.202Z0 -> 10.151.1.100 (2)
3902025-04-22T03:00:38.202Zresolver v4:
3912025-04-22T03:00:38.202Z10.151.1.100 -> 18:c0:4d:0d:9f:b2
3922025-04-22T03:00:38.202Z10.151.1.174 -> e6:9c:fd:ab:b1:26
3932025-04-22T03:00:38.202Zresolver v6:
3942025-04-22T03:00:38.202Zfe80::ac9b:65ff:fe57:f62a -> ae:9b:65:57:f6:2a
3952025-04-22T03:00:38.202Zfe80::a430:eff:fe86:b0ce -> a6:30:0e:86:b0:ce
3962025-04-22T03:00:38.202Znat v4:
3972025-04-22T03:00:38.202Znat_v6:
3982025-04-22T03:00:38.202Zport_mac:
3992025-04-22T03:00:38.202Z1000: a8:40:25:13:8d:92
4002025-04-22T03:00:38.203Z1: a8:40:25:13:8d:93
4012025-04-22T03:00:38.203Z2: a8:40:25:13:8d:b3
4022025-04-22T03:00:38.203Zproxy arp:
4032025-04-22T03:00:38.203Z10.151.1.164/10.151.1.227: a8:e1:de:01:70:1d
4042025-04-22T03:00:38.229Z+ retry=0
4052025-04-22T03:00:38.267Z+++ grep oxz_ntp
4062025-04-22T03:00:38.267Z+++ zoneadm list -n
4072025-04-22T03:00:38.292Z++ pfexec svcs -z -Hostate oxide/ntp
4082025-04-22T03:00:38.317Zsvcs: zone '-Hostate': entity not found
4092025-04-22T03:00:38.342Z++ true
4102025-04-22T03:00:38.342Z+ [[ '' != online ]]
4112025-04-22T03:00:38.342Z+ [[ 0 -gt 60 ]]
4122025-04-22T03:00:38.342Z+ sleep 1
4132025-04-22T03:00:39.336Z+ retry=1
4142025-04-22T03:00:39.360Z+++ zoneadm list -n
4152025-04-22T03:00:39.361Z+++ grep oxz_ntp
4162025-04-22T03:00:39.361Z++ pfexec svcs -z -Hostate oxide/ntp
4172025-04-22T03:00:39.361Zsvcs: zone '-Hostate': entity not found
4182025-04-22T03:00:39.361Z++ true
4192025-04-22T03:00:39.361Z+ [[ '' != online ]]
4202025-04-22T03:00:39.361Z+ [[ 1 -gt 60 ]]
4212025-04-22T03:00:39.361Z+ sleep 1
4222025-04-22T03:00:40.356Z+ retry=2
4232025-04-22T03:00:40.381Z+++ zoneadm list -n
4242025-04-22T03:00:40.382Z+++ grep oxz_ntp
4252025-04-22T03:00:40.382Z++ pfexec svcs -z -Hostate oxide/ntp
4262025-04-22T03:00:40.382Zsvcs: zone '-Hostate': entity not found
4272025-04-22T03:00:40.382Z++ true
4282025-04-22T03:00:40.382Z+ [[ '' != online ]]
4292025-04-22T03:00:40.382Z+ [[ 2 -gt 60 ]]
4302025-04-22T03:00:40.382Z+ sleep 1
4312025-04-22T03:00:41.380Z+ retry=3
4322025-04-22T03:00:41.405Z+++ zoneadm list -n
4332025-04-22T03:00:41.405Z+++ grep oxz_ntp
4342025-04-22T03:00:41.405Z++ pfexec svcs -z -Hostate oxide/ntp
4352025-04-22T03:00:41.405Zsvcs: zone '-Hostate': entity not found
4362025-04-22T03:00:41.405Z++ true
4372025-04-22T03:00:41.405Z+ [[ '' != online ]]
4382025-04-22T03:00:41.405Z+ [[ 3 -gt 60 ]]
4392025-04-22T03:00:41.405Z+ sleep 1
4402025-04-22T03:00:42.400Z+ retry=4
4412025-04-22T03:00:42.426Z+++ zoneadm list -n
4422025-04-22T03:00:42.426Z+++ grep oxz_ntp
4432025-04-22T03:00:42.426Z++ pfexec svcs -z -Hostate oxide/ntp
4442025-04-22T03:00:42.426Zsvcs: zone '-Hostate': entity not found
4452025-04-22T03:00:42.426Z++ true
4462025-04-22T03:00:42.426Z+ [[ '' != online ]]
4472025-04-22T03:00:42.426Z+ [[ 4 -gt 60 ]]
4482025-04-22T03:00:42.426Z+ sleep 1
4492025-04-22T03:00:43.421Z+ retry=5
4502025-04-22T03:00:43.446Z+++ zoneadm list -n
4512025-04-22T03:00:43.446Z+++ grep oxz_ntp
4522025-04-22T03:00:43.446Z++ pfexec svcs -z -Hostate oxide/ntp
4532025-04-22T03:00:43.446Zsvcs: zone '-Hostate': entity not found
4542025-04-22T03:00:43.446Z++ true
4552025-04-22T03:00:43.446Z+ [[ '' != online ]]
4562025-04-22T03:00:43.446Z+ [[ 5 -gt 60 ]]
4572025-04-22T03:00:43.446Z+ sleep 1
4582025-04-22T03:00:44.446Z+ retry=6
4592025-04-22T03:00:44.471Z+++ zoneadm list -n
4602025-04-22T03:00:44.471Z+++ grep oxz_ntp
4612025-04-22T03:00:44.472Z++ pfexec svcs -z -Hostate oxide/ntp
4622025-04-22T03:00:44.472Zsvcs: zone '-Hostate': entity not found
4632025-04-22T03:00:44.472Z++ true
4642025-04-22T03:00:44.472Z+ [[ '' != online ]]
4652025-04-22T03:00:44.472Z+ [[ 6 -gt 60 ]]
4662025-04-22T03:00:44.472Z+ sleep 1
4672025-04-22T03:00:45.468Z+ retry=7
4682025-04-22T03:00:45.493Z+++ zoneadm list -n
4692025-04-22T03:00:45.493Z+++ grep oxz_ntp
4702025-04-22T03:00:45.493Z++ pfexec svcs -z -Hostate oxide/ntp
4712025-04-22T03:00:45.493Zsvcs: zone '-Hostate': entity not found
4722025-04-22T03:00:45.493Z++ true
4732025-04-22T03:00:45.493Z+ [[ '' != online ]]
4742025-04-22T03:00:45.493Z+ [[ 7 -gt 60 ]]
4752025-04-22T03:00:45.493Z+ sleep 1
4762025-04-22T03:00:46.491Z+ retry=8
4772025-04-22T03:00:46.516Z+++ zoneadm list -n
4782025-04-22T03:00:46.516Z+++ grep oxz_ntp
4792025-04-22T03:00:46.516Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
4802025-04-22T03:00:46.541Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4812025-04-22T03:00:46.541Z++ true
4822025-04-22T03:00:46.541Z+ [[ '' != online ]]
4832025-04-22T03:00:46.541Z+ [[ 8 -gt 60 ]]
4842025-04-22T03:00:46.541Z+ sleep 1
4852025-04-22T03:00:47.523Z+ retry=9
4862025-04-22T03:00:47.548Z+++ zoneadm list -n
4872025-04-22T03:00:47.548Z+++ grep oxz_ntp
4882025-04-22T03:00:47.548Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
4892025-04-22T03:00:47.548Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4902025-04-22T03:00:47.573Z++ true
4912025-04-22T03:00:47.573Z+ [[ '' != online ]]
4922025-04-22T03:00:47.573Z+ [[ 9 -gt 60 ]]
4932025-04-22T03:00:47.573Z+ sleep 1
4942025-04-22T03:00:48.552Z+ retry=10
4952025-04-22T03:00:48.577Z+++ zoneadm list -n
4962025-04-22T03:00:48.577Z+++ grep oxz_ntp
4972025-04-22T03:00:48.577Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
4982025-04-22T03:00:48.603Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4992025-04-22T03:00:48.603Z++ true
5002025-04-22T03:00:48.603Z+ [[ '' != online ]]
5012025-04-22T03:00:48.603Z+ [[ 10 -gt 60 ]]
5022025-04-22T03:00:48.603Z+ sleep 1
5032025-04-22T03:00:49.583Z+ retry=11
5042025-04-22T03:00:49.608Z+++ zoneadm list -n
5052025-04-22T03:00:49.608Z+++ grep oxz_ntp
5062025-04-22T03:00:49.608Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5072025-04-22T03:00:49.608Z+ [[ offline != online ]]
5082025-04-22T03:00:49.608Z+ [[ 11 -gt 60 ]]
5092025-04-22T03:00:49.608Z+ sleep 1
5102025-04-22T03:00:50.605Z+ retry=12
5112025-04-22T03:00:50.638Z+++ zoneadm list -n
5122025-04-22T03:00:50.638Z+++ grep oxz_ntp
5132025-04-22T03:00:50.638Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5142025-04-22T03:00:50.638Z+ [[ offline != online ]]
5152025-04-22T03:00:50.638Z+ [[ 12 -gt 60 ]]
5162025-04-22T03:00:50.638Z+ sleep 1
5172025-04-22T03:00:51.628Z+ retry=13
5182025-04-22T03:00:51.653Z+++ zoneadm list -n
5192025-04-22T03:00:51.653Z+++ grep oxz_ntp
5202025-04-22T03:00:51.653Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5212025-04-22T03:00:51.653Z+ [[ offline != online ]]
5222025-04-22T03:00:51.653Z+ [[ 13 -gt 60 ]]
5232025-04-22T03:00:51.653Z+ sleep 1
5242025-04-22T03:00:52.649Z+ retry=14
5252025-04-22T03:00:52.673Z+++ zoneadm list -n
5262025-04-22T03:00:52.674Z+++ grep oxz_ntp
5272025-04-22T03:00:52.674Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5282025-04-22T03:00:52.674Z+ [[ offline != online ]]
5292025-04-22T03:00:52.674Z+ [[ 14 -gt 60 ]]
5302025-04-22T03:00:52.674Z+ sleep 1
5312025-04-22T03:00:53.671Z+ retry=15
5322025-04-22T03:00:53.696Z+++ zoneadm list -n
5332025-04-22T03:00:53.696Z+++ grep oxz_ntp
5342025-04-22T03:00:53.696Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5352025-04-22T03:00:53.696Z+ [[ offline != online ]]
5362025-04-22T03:00:53.696Z+ [[ 15 -gt 60 ]]
5372025-04-22T03:00:53.696Z+ sleep 1
5382025-04-22T03:00:54.692Z+ retry=16
5392025-04-22T03:00:54.717Z+++ zoneadm list -n
5402025-04-22T03:00:54.717Z+++ grep oxz_ntp
5412025-04-22T03:00:54.717Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5422025-04-22T03:00:54.717Z+ [[ offline != online ]]
5432025-04-22T03:00:54.717Z+ [[ 16 -gt 60 ]]
5442025-04-22T03:00:54.717Z+ sleep 1
5452025-04-22T03:00:55.713Z+ retry=17
5462025-04-22T03:00:55.738Z+++ zoneadm list -n
5472025-04-22T03:00:55.738Z+++ grep oxz_ntp
5482025-04-22T03:00:55.738Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5492025-04-22T03:00:55.738Z+ [[ offline != online ]]
5502025-04-22T03:00:55.738Z+ [[ 17 -gt 60 ]]
5512025-04-22T03:00:55.738Z+ sleep 1
5522025-04-22T03:00:56.735Z+ retry=18
5532025-04-22T03:00:56.760Z+++ zoneadm list -n
5542025-04-22T03:00:56.760Z+++ grep oxz_ntp
5552025-04-22T03:00:56.760Z++ pfexec svcs -z oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6 -Hostate oxide/ntp
5562025-04-22T03:00:56.760Z+ [[ online != online ]]
5572025-04-22T03:00:56.760Z+ echo 'Waited for chrony: 18s'
5582025-04-22T03:00:56.760Z+ retry=0
5592025-04-22T03:00:56.760ZWaited for chrony: 18s
5602025-04-22T03:00:56.760Z+ zoneadm list
5612025-04-22T03:00:56.760Z+ grep nexus
5622025-04-22T03:00:56.786Z+ [[ 0 -gt 300 ]]
5632025-04-22T03:00:56.786Z+ sleep 1
 
5642025-04-22T03:00:56.788Zbuskin console login: Apr 22 03:00:55 buskin ip: ip: joining multicasts failed (4) on opte0 - will use link layer broadcasts for multicast
 
5652025-04-22T03:00:57.763Z+ retry=1
5662025-04-22T03:00:57.788Z+ zoneadm list
5672025-04-22T03:00:57.788Z+ grep nexus
5682025-04-22T03:00:57.788Z+ [[ 1 -gt 300 ]]
5692025-04-22T03:00:57.788Z+ sleep 1
5702025-04-22T03:00:58.773Z+ retry=2
5712025-04-22T03:00:58.798Z+ zoneadm list
5722025-04-22T03:00:58.798Z+ grep nexus
5732025-04-22T03:00:58.798Z+ [[ 2 -gt 300 ]]
5742025-04-22T03:00:58.798Z+ sleep 1
5752025-04-22T03:00:59.784Z+ retry=3
5762025-04-22T03:00:59.809Z+ zoneadm list
5772025-04-22T03:00:59.810Z+ grep nexus
5782025-04-22T03:00:59.810Z+ [[ 3 -gt 300 ]]
5792025-04-22T03:00:59.810Z+ sleep 1
5802025-04-22T03:01:00.795Z+ retry=4
5812025-04-22T03:01:00.820Z+ zoneadm list
5822025-04-22T03:01:00.820Z+ grep nexus
5832025-04-22T03:01:00.820Z+ [[ 4 -gt 300 ]]
5842025-04-22T03:01:00.820Z+ sleep 1
5852025-04-22T03:01:01.805Z+ retry=5
5862025-04-22T03:01:01.830Z+ zoneadm list
5872025-04-22T03:01:01.830Z+ grep nexus
5882025-04-22T03:01:01.831Z+ [[ 5 -gt 300 ]]
5892025-04-22T03:01:01.831Z+ sleep 1
5902025-04-22T03:01:02.816Z+ retry=6
5912025-04-22T03:01:02.841Z+ zoneadm list
5922025-04-22T03:01:02.841Z+ grep nexus
5932025-04-22T03:01:02.841Z+ [[ 6 -gt 300 ]]
5942025-04-22T03:01:02.841Z+ sleep 1
5952025-04-22T03:01:03.826Z+ retry=7
5962025-04-22T03:01:03.853Z+ zoneadm list
5972025-04-22T03:01:03.853Z+ grep nexus
5982025-04-22T03:01:03.853Z+ [[ 7 -gt 300 ]]
5992025-04-22T03:01:03.853Z+ sleep 1
6002025-04-22T03:01:04.838Z+ retry=8
6012025-04-22T03:01:04.863Z+ zoneadm list
6022025-04-22T03:01:04.863Z+ grep nexus
6032025-04-22T03:01:04.863Z+ [[ 8 -gt 300 ]]
6042025-04-22T03:01:04.863Z+ sleep 1
6052025-04-22T03:01:05.847Z+ retry=9
6062025-04-22T03:01:05.873Z+ zoneadm list
6072025-04-22T03:01:05.873Z+ grep nexus
6082025-04-22T03:01:05.873Z+ [[ 9 -gt 300 ]]
6092025-04-22T03:01:05.873Z+ sleep 1
6102025-04-22T03:01:06.858Z+ retry=10
6112025-04-22T03:01:06.883Z+ zoneadm list
6122025-04-22T03:01:06.883Z+ grep nexus
6132025-04-22T03:01:06.883Z+ [[ 10 -gt 300 ]]
6142025-04-22T03:01:06.883Z+ sleep 1
6152025-04-22T03:01:07.868Z+ retry=11
6162025-04-22T03:01:07.893Z+ zoneadm list
6172025-04-22T03:01:07.893Z+ grep nexus
6182025-04-22T03:01:07.893Z+ [[ 11 -gt 300 ]]
6192025-04-22T03:01:07.893Z+ sleep 1
6202025-04-22T03:01:08.878Z+ retry=12
6212025-04-22T03:01:08.903Z+ zoneadm list
6222025-04-22T03:01:08.903Z+ grep nexus
6232025-04-22T03:01:08.903Z+ [[ 12 -gt 300 ]]
6242025-04-22T03:01:08.903Z+ sleep 1
6252025-04-22T03:01:09.889Z+ retry=13
6262025-04-22T03:01:09.916Z+ zoneadm list
6272025-04-22T03:01:09.916Z+ grep nexus
6282025-04-22T03:01:09.916Z+ [[ 13 -gt 300 ]]
6292025-04-22T03:01:09.916Z+ sleep 1
6302025-04-22T03:01:10.900Z+ retry=14
6312025-04-22T03:01:10.925Z+ zoneadm list
6322025-04-22T03:01:10.925Z+ grep nexus
6332025-04-22T03:01:10.925Z+ [[ 14 -gt 300 ]]
6342025-04-22T03:01:10.925Z+ sleep 1
6352025-04-22T03:01:11.909Z+ retry=15
6362025-04-22T03:01:11.935Z+ zoneadm list
6372025-04-22T03:01:11.935Z+ grep nexus
6382025-04-22T03:01:11.935Z+ [[ 15 -gt 300 ]]
6392025-04-22T03:01:11.935Z+ sleep 1
6402025-04-22T03:01:12.921Z+ retry=16
6412025-04-22T03:01:12.946Z+ zoneadm list
6422025-04-22T03:01:12.946Z+ grep nexus
6432025-04-22T03:01:12.946Z+ [[ 16 -gt 300 ]]
6442025-04-22T03:01:12.947Z+ sleep 1
6452025-04-22T03:01:13.937Z+ retry=17
6462025-04-22T03:01:13.962Z+ zoneadm list
6472025-04-22T03:01:13.962Z+ grep nexus
6482025-04-22T03:01:13.963Z+ [[ 17 -gt 300 ]]
6492025-04-22T03:01:13.963Z+ sleep 1
6502025-04-22T03:01:14.948Z+ retry=18
6512025-04-22T03:01:14.997Z+ grep nexus
6522025-04-22T03:01:15.056Z+ zoneadm list
6532025-04-22T03:01:15.083Z+ [[ 18 -gt 300 ]]
6542025-04-22T03:01:15.083Z+ sleep 1
6552025-04-22T03:01:16.081Z+ retry=19
6562025-04-22T03:01:16.106Z+ zoneadm list
6572025-04-22T03:01:16.106Z+ grep nexus
6582025-04-22T03:01:16.107Z+ [[ 19 -gt 300 ]]
6592025-04-22T03:01:16.107Z+ sleep 1
6602025-04-22T03:01:17.098Z+ retry=20
6612025-04-22T03:01:17.123Z+ zoneadm list
6622025-04-22T03:01:17.123Z+ grep nexus
6632025-04-22T03:01:17.123Z+ [[ 20 -gt 300 ]]
6642025-04-22T03:01:17.123Z+ sleep 1
6652025-04-22T03:01:18.163Z+ retry=21
6662025-04-22T03:01:18.164Z+ zoneadm list
6672025-04-22T03:01:18.164Z+ grep nexus
6682025-04-22T03:01:18.164Z+ [[ 21 -gt 300 ]]
6692025-04-22T03:01:18.164Z+ sleep 1
6702025-04-22T03:01:19.132Z+ retry=22
6712025-04-22T03:01:19.157Z+ zoneadm list
6722025-04-22T03:01:19.157Z+ grep nexus
6732025-04-22T03:01:19.157Z+ [[ 22 -gt 300 ]]
6742025-04-22T03:01:19.157Z+ sleep 1
6752025-04-22T03:01:20.147Z+ retry=23
6762025-04-22T03:01:20.172Z+ zoneadm list
6772025-04-22T03:01:20.173Z+ grep nexus
6782025-04-22T03:01:20.173Z+ [[ 23 -gt 300 ]]
6792025-04-22T03:01:20.173Z+ sleep 1
6802025-04-22T03:01:21.160Z+ retry=24
6812025-04-22T03:01:21.185Z+ zoneadm list
6822025-04-22T03:01:21.185Z+ grep nexus
6832025-04-22T03:01:21.185Z+ [[ 24 -gt 300 ]]
6842025-04-22T03:01:21.185Z+ sleep 1
6852025-04-22T03:01:22.172Z+ retry=25
6862025-04-22T03:01:22.197Z+ zoneadm list
6872025-04-22T03:01:22.197Z+ grep nexus
6882025-04-22T03:01:22.197Z+ [[ 25 -gt 300 ]]
6892025-04-22T03:01:22.197Z+ sleep 1
6902025-04-22T03:01:23.184Z+ retry=26
6912025-04-22T03:01:23.209Z+ zoneadm list
6922025-04-22T03:01:23.209Z+ grep nexus
6932025-04-22T03:01:23.209Z+ [[ 26 -gt 300 ]]
6942025-04-22T03:01:23.209Z+ sleep 1
6952025-04-22T03:01:24.195Z+ retry=27
6962025-04-22T03:01:24.220Z+ zoneadm list
6972025-04-22T03:01:24.220Z+ grep nexus
6982025-04-22T03:01:24.221Z+ [[ 27 -gt 300 ]]
6992025-04-22T03:01:24.221Z+ sleep 1
7002025-04-22T03:01:25.208Z+ retry=28
7012025-04-22T03:01:25.233Z+ zoneadm list
7022025-04-22T03:01:25.233Z+ grep nexus
7032025-04-22T03:01:25.233Z+ [[ 28 -gt 300 ]]
7042025-04-22T03:01:25.233Z+ sleep 1
7052025-04-22T03:01:26.221Z+ retry=29
7062025-04-22T03:01:26.246Z+ zoneadm list
7072025-04-22T03:01:26.246Z+ grep nexus
7082025-04-22T03:01:26.247Z+ [[ 29 -gt 300 ]]
7092025-04-22T03:01:26.247Z+ sleep 1
7102025-04-22T03:01:27.235Z+ retry=30
7112025-04-22T03:01:27.260Z+ zoneadm list
7122025-04-22T03:01:27.260Z+ grep nexus
7132025-04-22T03:01:27.260Z+ [[ 30 -gt 300 ]]
7142025-04-22T03:01:27.260Z+ sleep 1
7152025-04-22T03:01:33.046Z+ retry=31
7162025-04-22T03:01:33.087Z+ zoneadm list
7172025-04-22T03:01:33.112Z+ grep nexus
7182025-04-22T03:01:33.137Z+ [[ 31 -gt 300 ]]
7192025-04-22T03:01:33.137Z+ sleep 1
 
7202025-04-22T03:01:33.403ZApr 22 03:01:33 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:99.28.14.242:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
7212025-04-22T03:01:33.436ZApr 22 03:01:33 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:216.66.48.42:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
 
7222025-04-22T03:01:34.135Z+ retry=32
7232025-04-22T03:01:34.160Z+ zoneadm list
7242025-04-22T03:01:34.160Z+ grep nexus
7252025-04-22T03:01:34.160Z+ [[ 32 -gt 300 ]]
7262025-04-22T03:01:34.160Z+ sleep 1
7272025-04-22T03:01:35.202Z+ retry=33
7282025-04-22T03:01:35.307Z+ zoneadm list
7292025-04-22T03:01:35.717Z+ grep nexus
7302025-04-22T03:01:35.987Z+ [[ 33 -gt 300 ]]
7312025-04-22T03:01:36.012Z+ sleep 1
7322025-04-22T03:01:37.067Z+ retry=34
7332025-04-22T03:01:37.092Z+ zoneadm list
7342025-04-22T03:01:37.093Z+ grep nexus
7352025-04-22T03:01:37.093Z+ [[ 34 -gt 300 ]]
7362025-04-22T03:01:37.093Z+ sleep 1
7372025-04-22T03:01:38.088Z+ retry=35
7382025-04-22T03:01:38.113Z+ zoneadm list
7392025-04-22T03:01:38.113Z+ grep nexus
7402025-04-22T03:01:38.113Z+ [[ 35 -gt 300 ]]
7412025-04-22T03:01:38.113Z+ sleep 1
7422025-04-22T03:01:39.105Z+ retry=36
7432025-04-22T03:01:39.130Z+ zoneadm list
7442025-04-22T03:01:39.131Z+ grep nexus
7452025-04-22T03:01:39.131Z+ [[ 36 -gt 300 ]]
7462025-04-22T03:01:39.131Z+ sleep 1
7472025-04-22T03:01:40.123Z+ retry=37
7482025-04-22T03:01:40.148Z+ zoneadm list
7492025-04-22T03:01:40.148Z+ grep nexus
7502025-04-22T03:01:40.148Z+ [[ 37 -gt 300 ]]
7512025-04-22T03:01:40.149Z+ sleep 1
7522025-04-22T03:01:41.140Z+ retry=38
7532025-04-22T03:01:41.166Z+ zoneadm list
7542025-04-22T03:01:41.166Z+ grep nexus
7552025-04-22T03:01:41.191Z+ [[ 38 -gt 300 ]]
7562025-04-22T03:01:41.191Z+ sleep 1
7572025-04-22T03:01:42.301Z+ retry=39
7582025-04-22T03:01:42.326Z+ zoneadm list
7592025-04-22T03:01:42.326Z+ grep nexus
7602025-04-22T03:01:42.326Z+ [[ 39 -gt 300 ]]
7612025-04-22T03:01:42.326Z+ sleep 1
7622025-04-22T03:01:43.320Z+ retry=40
7632025-04-22T03:01:43.348Z+ zoneadm list
7642025-04-22T03:01:43.348Z+ grep nexus
7652025-04-22T03:01:43.348Z+ [[ 40 -gt 300 ]]
7662025-04-22T03:01:43.348Z+ sleep 1
7672025-04-22T03:01:44.337Z+ retry=41
7682025-04-22T03:01:44.363Z+ zoneadm list
7692025-04-22T03:01:44.363Z+ grep nexus
7702025-04-22T03:01:44.363Z+ [[ 41 -gt 300 ]]
7712025-04-22T03:01:44.363Z+ sleep 1
7722025-04-22T03:01:45.352Z+ retry=42
7732025-04-22T03:01:45.377Z+ zoneadm list
7742025-04-22T03:01:45.377Z+ grep nexus
7752025-04-22T03:01:45.377Z+ [[ 42 -gt 300 ]]
7762025-04-22T03:01:45.377Z+ sleep 1
7772025-04-22T03:01:46.365Z+ retry=43
7782025-04-22T03:01:46.390Z+ zoneadm list
7792025-04-22T03:01:46.390Z+ grep nexus
7802025-04-22T03:01:46.390Z+ [[ 43 -gt 300 ]]
7812025-04-22T03:01:46.390Z+ sleep 1
7822025-04-22T03:01:47.377Z+ retry=44
7832025-04-22T03:01:47.403Z+ zoneadm list
7842025-04-22T03:01:47.403Z+ grep nexus
7852025-04-22T03:01:47.403Z+ [[ 44 -gt 300 ]]
7862025-04-22T03:01:47.403Z+ sleep 1
7872025-04-22T03:01:48.388Z+ retry=45
7882025-04-22T03:01:48.414Z+ zoneadm list
7892025-04-22T03:01:48.414Z+ grep nexus
7902025-04-22T03:01:48.414Z+ [[ 45 -gt 300 ]]
7912025-04-22T03:01:48.414Z+ sleep 1
7922025-04-22T03:01:49.400Z+ retry=46
7932025-04-22T03:01:49.425Z+ zoneadm list
7942025-04-22T03:01:49.425Z+ grep nexus
7952025-04-22T03:01:49.425Z+ [[ 46 -gt 300 ]]
7962025-04-22T03:01:49.425Z+ sleep 1
7972025-04-22T03:01:50.412Z+ retry=47
7982025-04-22T03:01:50.463Z+ zoneadm list
7992025-04-22T03:01:50.463Z+ grep nexus
8002025-04-22T03:01:50.463Z+ [[ 47 -gt 300 ]]
8012025-04-22T03:01:50.463Z+ sleep 1
8022025-04-22T03:01:51.424Z+ retry=48
8032025-04-22T03:01:51.449Z+ zoneadm list
8042025-04-22T03:01:51.449Z+ grep nexus
8052025-04-22T03:01:51.449Z+ [[ 48 -gt 300 ]]
8062025-04-22T03:01:51.449Z+ sleep 1
8072025-04-22T03:01:52.437Z+ retry=49
8082025-04-22T03:01:52.462Z+ zoneadm list
8092025-04-22T03:01:52.463Z+ grep nexus
8102025-04-22T03:01:52.463Z+ [[ 49 -gt 300 ]]
8112025-04-22T03:01:52.463Z+ sleep 1
8122025-04-22T03:01:53.449Z+ retry=50
8132025-04-22T03:01:53.473Z+ zoneadm list
8142025-04-22T03:01:53.474Z+ grep nexus
8152025-04-22T03:01:53.474Z+ [[ 50 -gt 300 ]]
8162025-04-22T03:01:53.474Z+ sleep 1
8172025-04-22T03:01:54.461Z+ retry=51
8182025-04-22T03:01:54.486Z+ zoneadm list
8192025-04-22T03:01:54.486Z+ grep nexus
8202025-04-22T03:01:54.486Z+ [[ 51 -gt 300 ]]
8212025-04-22T03:01:54.486Z+ sleep 1
8222025-04-22T03:01:55.474Z+ retry=52
8232025-04-22T03:01:55.499Z+ zoneadm list
8242025-04-22T03:01:55.499Z+ grep nexus
8252025-04-22T03:01:55.499Z+ [[ 52 -gt 300 ]]
8262025-04-22T03:01:55.499Z+ sleep 1
8272025-04-22T03:01:56.487Z+ retry=53
8282025-04-22T03:01:56.512Z+ zoneadm list
8292025-04-22T03:01:56.512Z+ grep nexus
8302025-04-22T03:01:56.512Z+ [[ 53 -gt 300 ]]
8312025-04-22T03:01:56.512Z+ sleep 1
8322025-04-22T03:01:57.500Z+ retry=54
8332025-04-22T03:01:57.525Z+ zoneadm list
8342025-04-22T03:01:57.525Z+ grep nexus
8352025-04-22T03:01:57.525Z+ [[ 54 -gt 300 ]]
8362025-04-22T03:01:57.525Z+ sleep 1
8372025-04-22T03:01:58.513Z+ retry=55
8382025-04-22T03:01:58.538Z+ zoneadm list
8392025-04-22T03:01:58.538Z+ grep nexus
8402025-04-22T03:01:58.538Z+ [[ 55 -gt 300 ]]
8412025-04-22T03:01:58.538Z+ sleep 1
8422025-04-22T03:01:59.526Z+ retry=56
8432025-04-22T03:01:59.551Z+ zoneadm list
8442025-04-22T03:01:59.551Z+ grep nexus
8452025-04-22T03:01:59.551Z+ [[ 56 -gt 300 ]]
8462025-04-22T03:01:59.551Z+ sleep 1
8472025-04-22T03:02:00.539Z+ retry=57
8482025-04-22T03:02:00.564Z+ zoneadm list
8492025-04-22T03:02:00.564Z+ grep nexus
8502025-04-22T03:02:00.564Z+ [[ 57 -gt 300 ]]
8512025-04-22T03:02:00.564Z+ sleep 1
8522025-04-22T03:02:01.552Z+ retry=58
8532025-04-22T03:02:01.577Z+ zoneadm list
8542025-04-22T03:02:01.577Z+ grep nexus
8552025-04-22T03:02:01.577Z+ [[ 58 -gt 300 ]]
8562025-04-22T03:02:01.577Z+ sleep 1
8572025-04-22T03:02:02.566Z+ retry=59
8582025-04-22T03:02:02.591Z+ zoneadm list
8592025-04-22T03:02:02.591Z+ grep nexus
8602025-04-22T03:02:02.591Z+ [[ 59 -gt 300 ]]
8612025-04-22T03:02:02.591Z+ sleep 1
8622025-04-22T03:02:03.579Z+ retry=60
8632025-04-22T03:02:03.604Z+ zoneadm list
8642025-04-22T03:02:03.604Z+ grep nexus
8652025-04-22T03:02:03.604Z+ [[ 60 -gt 300 ]]
8662025-04-22T03:02:03.604Z+ sleep 1
8672025-04-22T03:02:04.593Z+ retry=61
8682025-04-22T03:02:04.617Z+ zoneadm list
8692025-04-22T03:02:04.618Z+ grep nexus
8702025-04-22T03:02:04.618Z+ [[ 61 -gt 300 ]]
8712025-04-22T03:02:04.618Z+ sleep 1
8722025-04-22T03:02:05.606Z+ retry=62
8732025-04-22T03:02:05.630Z+ zoneadm list
8742025-04-22T03:02:05.631Z+ grep nexus
8752025-04-22T03:02:05.631Z+ [[ 62 -gt 300 ]]
8762025-04-22T03:02:05.631Z+ sleep 1
8772025-04-22T03:02:06.619Z+ retry=63
8782025-04-22T03:02:06.645Z+ zoneadm list
8792025-04-22T03:02:06.645Z+ grep nexus
8802025-04-22T03:02:06.645Z+ [[ 63 -gt 300 ]]
8812025-04-22T03:02:06.645Z+ sleep 1
8822025-04-22T03:02:07.641Z+ retry=64
8832025-04-22T03:02:07.666Z+ zoneadm list
8842025-04-22T03:02:07.666Z+ grep nexus
8852025-04-22T03:02:07.666Z+ [[ 64 -gt 300 ]]
8862025-04-22T03:02:07.666Z+ sleep 1
8872025-04-22T03:02:08.662Z+ retry=65
8882025-04-22T03:02:08.687Z+ zoneadm list
8892025-04-22T03:02:08.687Z+ grep nexus
8902025-04-22T03:02:08.687Z+ [[ 65 -gt 300 ]]
8912025-04-22T03:02:08.687Z+ sleep 1
8922025-04-22T03:02:09.681Z+ retry=66
8932025-04-22T03:02:09.707Z+ zoneadm list
8942025-04-22T03:02:09.707Z+ grep nexus
8952025-04-22T03:02:09.707Z+ [[ 66 -gt 300 ]]
8962025-04-22T03:02:09.707Z+ sleep 1
8972025-04-22T03:02:10.713Z+ retry=67
8982025-04-22T03:02:10.739Z+ zoneadm list
8992025-04-22T03:02:10.739Z+ grep nexus
9002025-04-22T03:02:10.764Z+ [[ 67 -gt 300 ]]
9012025-04-22T03:02:10.764Z+ sleep 1
9022025-04-22T03:02:17.818Z+ retry=68
9032025-04-22T03:02:18.105Z+ zoneadm list
9042025-04-22T03:02:18.105Z+ grep nexus
9052025-04-22T03:02:18.105Z+ [[ 68 -gt 300 ]]
9062025-04-22T03:02:18.105Z+ sleep 1
 
9072025-04-22T03:02:18.557ZApr 22 03:02:17 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:205.233.73.201:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
9082025-04-22T03:02:18.589ZApr 22 03:02:17 buskin last message repeated 2 times
9092025-04-22T03:02:18.624ZApr 22 03:02:17 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:23.186.168.132:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
9102025-04-22T03:02:18.659ZApr 22 03:02:17 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:99.28.14.242:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
9112025-04-22T03:02:18.697ZApr 22 03:02:17 buskin last message repeated 1 time
9122025-04-22T03:02:18.730ZApr 22 03:02:17 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:216.66.48.42:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
 
9132025-04-22T03:02:19.249Z+ retry=69
9142025-04-22T03:02:19.275Z+ zoneadm list
9152025-04-22T03:02:19.275Z+ grep nexus
9162025-04-22T03:02:19.275Z+ [[ 69 -gt 300 ]]
9172025-04-22T03:02:19.275Z+ sleep 1
9182025-04-22T03:02:20.269Z+ retry=70
9192025-04-22T03:02:20.294Z+ zoneadm list
9202025-04-22T03:02:20.294Z+ grep nexus
9212025-04-22T03:02:20.294Z+ [[ 70 -gt 300 ]]
9222025-04-22T03:02:20.294Z+ sleep 1
9232025-04-22T03:02:21.301Z+ retry=71
9242025-04-22T03:02:21.334Z+ zoneadm list
9252025-04-22T03:02:21.334Z+ grep nexus
9262025-04-22T03:02:21.334Z+ [[ 71 -gt 300 ]]
9272025-04-22T03:02:21.334Z+ sleep 1
9282025-04-22T03:02:22.325Z+ retry=72
9292025-04-22T03:02:22.350Z+ zoneadm list
9302025-04-22T03:02:22.350Z+ grep nexus
9312025-04-22T03:02:22.350Z+ [[ 72 -gt 300 ]]
9322025-04-22T03:02:22.350Z+ sleep 1
9332025-04-22T03:02:23.354Z+ retry=73
9342025-04-22T03:02:23.379Z+ zoneadm list
9352025-04-22T03:02:23.379Z+ grep nexus
9362025-04-22T03:02:23.379Z+ [[ 73 -gt 300 ]]
9372025-04-22T03:02:23.379Z+ sleep 1
9382025-04-22T03:02:24.382Z+ retry=74
9392025-04-22T03:02:24.407Z+ zoneadm list
9402025-04-22T03:02:24.407Z+ grep nexus
9412025-04-22T03:02:24.407Zoxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e
9422025-04-22T03:02:24.407Zoxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77
9432025-04-22T03:02:24.407Zoxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c
9442025-04-22T03:02:24.407Z+ echo 'Waited for nexus: 74s'
9452025-04-22T03:02:24.407ZWaited for nexus: 74s
9462025-04-22T03:02:24.407Z+ retry=0
9472025-04-22T03:02:24.407Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9482025-04-22T03:02:24.432Z+ [[ 0 -gt 300 ]]
9492025-04-22T03:02:24.432Z+ sleep 1
9502025-04-22T03:02:25.420Z+ retry=1
9512025-04-22T03:02:25.420Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9522025-04-22T03:02:25.470Z+ [[ 1 -gt 300 ]]
9532025-04-22T03:02:25.470Z+ sleep 1
9542025-04-22T03:02:26.432Z+ retry=2
9552025-04-22T03:02:26.432Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9562025-04-22T03:02:26.457Z+ [[ 2 -gt 300 ]]
9572025-04-22T03:02:26.458Z+ sleep 1
9582025-04-22T03:02:27.445Z+ retry=3
9592025-04-22T03:02:27.445Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9602025-04-22T03:02:27.470Z+ [[ 3 -gt 300 ]]
9612025-04-22T03:02:27.470Z+ sleep 1
9622025-04-22T03:02:28.459Z+ retry=4
9632025-04-22T03:02:28.459Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9642025-04-22T03:02:28.484Z+ [[ 4 -gt 300 ]]
9652025-04-22T03:02:28.484Z+ sleep 1
 
9662025-04-22T03:02:29.085ZApr 22 03:02:17 buskin last message repeated 1 time
9672025-04-22T03:02:29.120ZApr 22 03:02:28 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:205.233.73.201:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
 
9682025-04-22T03:02:29.469Z+ retry=5
9692025-04-22T03:02:29.469Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9702025-04-22T03:02:29.494Z+ [[ 5 -gt 300 ]]
9712025-04-22T03:02:29.495Z+ sleep 1
9722025-04-22T03:02:30.478Z+ retry=6
9732025-04-22T03:02:30.478Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9742025-04-22T03:02:30.503Z+ [[ 6 -gt 300 ]]
9752025-04-22T03:02:30.503Z+ sleep 1
9762025-04-22T03:02:31.488Z+ retry=7
9772025-04-22T03:02:31.488Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9782025-04-22T03:02:31.513Z+ [[ 7 -gt 300 ]]
9792025-04-22T03:02:31.514Z+ sleep 1
9802025-04-22T03:02:32.499Z+ retry=8
9812025-04-22T03:02:32.499Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9822025-04-22T03:02:32.524Z+ [[ 8 -gt 300 ]]
9832025-04-22T03:02:32.524Z+ sleep 1
9842025-04-22T03:02:33.509Z+ retry=9
9852025-04-22T03:02:33.509Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9862025-04-22T03:02:33.534Z+ [[ 9 -gt 300 ]]
9872025-04-22T03:02:33.534Z+ sleep 1
9882025-04-22T03:02:34.520Z+ retry=10
9892025-04-22T03:02:34.520Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9902025-04-22T03:02:34.545Z+ [[ 10 -gt 300 ]]
9912025-04-22T03:02:34.546Z+ sleep 1
9922025-04-22T03:02:35.530Z+ retry=11
9932025-04-22T03:02:35.530Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9942025-04-22T03:02:35.555Z+ [[ 11 -gt 300 ]]
9952025-04-22T03:02:35.555Z+ sleep 1
9962025-04-22T03:02:36.541Z+ retry=12
9972025-04-22T03:02:36.541Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
9982025-04-22T03:02:36.566Z+ [[ 12 -gt 300 ]]
9992025-04-22T03:02:36.566Z+ sleep 1
10002025-04-22T03:02:37.552Z+ retry=13
10012025-04-22T03:02:37.552Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10022025-04-22T03:02:37.577Z+ [[ 13 -gt 300 ]]
10032025-04-22T03:02:37.577Z+ sleep 1
10042025-04-22T03:02:38.565Z+ retry=14
10052025-04-22T03:02:38.565Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10062025-04-22T03:02:38.590Z+ [[ 14 -gt 300 ]]
10072025-04-22T03:02:38.590Z+ sleep 1
10082025-04-22T03:02:39.581Z+ retry=15
10092025-04-22T03:02:39.581Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10102025-04-22T03:02:39.606Z+ [[ 15 -gt 300 ]]
10112025-04-22T03:02:39.606Z+ sleep 1
10122025-04-22T03:02:40.593Z+ retry=16
10132025-04-22T03:02:40.593Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10142025-04-22T03:02:40.618Z+ [[ 16 -gt 300 ]]
10152025-04-22T03:02:40.618Z+ sleep 1
10162025-04-22T03:02:41.603Z+ retry=17
10172025-04-22T03:02:41.603Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10182025-04-22T03:02:41.629Z+ [[ 17 -gt 300 ]]
10192025-04-22T03:02:41.629Z+ sleep 1
10202025-04-22T03:02:42.613Z+ retry=18
10212025-04-22T03:02:42.613Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10222025-04-22T03:02:42.639Z+ [[ 18 -gt 300 ]]
10232025-04-22T03:02:42.639Z+ sleep 1
10242025-04-22T03:02:43.625Z+ retry=19
10252025-04-22T03:02:43.625Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10262025-04-22T03:02:43.650Z+ [[ 19 -gt 300 ]]
10272025-04-22T03:02:43.650Z+ sleep 1
10282025-04-22T03:02:44.635Z+ retry=20
10292025-04-22T03:02:44.635Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10302025-04-22T03:02:44.660Z+ [[ 20 -gt 300 ]]
10312025-04-22T03:02:44.660Z+ sleep 1
10322025-04-22T03:02:45.646Z+ retry=21
10332025-04-22T03:02:45.646Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10342025-04-22T03:02:45.671Z+ [[ 21 -gt 300 ]]
10352025-04-22T03:02:45.671Z+ sleep 1
10362025-04-22T03:02:46.657Z+ retry=22
10372025-04-22T03:02:46.657Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10382025-04-22T03:02:46.682Z+ [[ 22 -gt 300 ]]
10392025-04-22T03:02:46.682Z+ sleep 1
10402025-04-22T03:02:47.667Z+ retry=23
10412025-04-22T03:02:47.667Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10422025-04-22T03:02:47.692Z+ [[ 23 -gt 300 ]]
10432025-04-22T03:02:47.692Z+ sleep 1
10442025-04-22T03:02:48.678Z+ retry=24
10452025-04-22T03:02:48.678Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10462025-04-22T03:02:48.703Z+ [[ 24 -gt 300 ]]
10472025-04-22T03:02:48.703Z+ sleep 1
 
10482025-04-22T03:02:49.561ZApr 22 03:02:48 buskin ip: ip: joining multicasts failed (4) on opte3 - will use link layer broadcasts for multicast
10492025-04-22T03:02:49.598ZApr 22 03:02:49 buskin ip: ip: joining multicasts failed (4) on opte4 - will use link layer broadcasts for multicast
10502025-04-22T03:02:49.632ZApr 22 03:02:49 buskin ip: ip: joining multicasts failed (4) on opte1 - will use link layer broadcasts for multicast
10512025-04-22T03:02:49.666ZApr 22 03:02:49 buskin ip: ip: joining multicasts failed (4) on opte2 - will use link layer broadcasts for multicast
 
10522025-04-22T03:02:49.690Z+ retry=25
10532025-04-22T03:02:49.690Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10542025-04-22T03:02:49.715Z+ [[ 25 -gt 300 ]]
10552025-04-22T03:02:49.715Z+ sleep 1
 
10562025-04-22T03:02:50.263ZApr 22 03:02:49 buskin ip: ip: joining multicasts failed (4) on opte5 - will use link layer broadcasts for multicast
 
10572025-04-22T03:02:50.701Z+ retry=26
10582025-04-22T03:02:50.701Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10592025-04-22T03:02:50.726Z+ [[ 26 -gt 300 ]]
10602025-04-22T03:02:50.726Z+ sleep 1
10612025-04-22T03:02:51.713Z+ retry=27
10622025-04-22T03:02:51.713Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10632025-04-22T03:02:51.738Z+ [[ 27 -gt 300 ]]
10642025-04-22T03:02:51.738Z+ sleep 1
10652025-04-22T03:02:52.724Z+ retry=28
10662025-04-22T03:02:52.724Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10672025-04-22T03:02:52.749Z+ [[ 28 -gt 300 ]]
10682025-04-22T03:02:52.749Z+ sleep 1
10692025-04-22T03:02:53.734Z+ retry=29
10702025-04-22T03:02:53.734Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10712025-04-22T03:02:53.759Z+ [[ 29 -gt 300 ]]
10722025-04-22T03:02:53.760Z+ sleep 1
10732025-04-22T03:02:54.745Z+ retry=30
10742025-04-22T03:02:54.746Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10752025-04-22T03:02:54.777Z+ [[ 30 -gt 300 ]]
10762025-04-22T03:02:54.778Z+ sleep 1
10772025-04-22T03:02:55.756Z+ retry=31
10782025-04-22T03:02:55.756Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10792025-04-22T03:02:55.783Z+ [[ 31 -gt 300 ]]
10802025-04-22T03:02:55.783Z+ sleep 1
10812025-04-22T03:02:56.768Z+ retry=32
10822025-04-22T03:02:56.768Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10832025-04-22T03:02:56.794Z+ [[ 32 -gt 300 ]]
10842025-04-22T03:02:56.794Z+ sleep 1
10852025-04-22T03:02:57.786Z+ retry=33
10862025-04-22T03:02:57.786Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10872025-04-22T03:02:57.811Z+ [[ 33 -gt 300 ]]
10882025-04-22T03:02:57.811Z+ sleep 1
10892025-04-22T03:02:58.797Z+ retry=34
10902025-04-22T03:02:58.797Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10912025-04-22T03:02:58.823Z+ [[ 34 -gt 300 ]]
10922025-04-22T03:02:58.823Z+ sleep 1
10932025-04-22T03:02:59.808Z+ retry=35
10942025-04-22T03:02:59.808Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10952025-04-22T03:02:59.834Z+ [[ 35 -gt 300 ]]
10962025-04-22T03:02:59.834Z+ sleep 1
10972025-04-22T03:03:00.820Z+ retry=36
10982025-04-22T03:03:00.820Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
10992025-04-22T03:03:00.845Z+ [[ 36 -gt 300 ]]
11002025-04-22T03:03:00.845Z+ sleep 1
11012025-04-22T03:03:01.831Z+ retry=37
11022025-04-22T03:03:01.831Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
11032025-04-22T03:03:01.856Z+ [[ 37 -gt 300 ]]
11042025-04-22T03:03:01.856Z+ sleep 1
11052025-04-22T03:03:02.842Z+ retry=38
11062025-04-22T03:03:02.842Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
11072025-04-22T03:03:02.867Z+ [[ 38 -gt 300 ]]
11082025-04-22T03:03:02.867Z+ sleep 1
11092025-04-22T03:03:03.853Z+ retry=39
11102025-04-22T03:03:03.853Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
11112025-04-22T03:03:03.878Z+ [[ 39 -gt 300 ]]
11122025-04-22T03:03:03.878Z+ sleep 1
11132025-04-22T03:03:04.864Z+ retry=40
11142025-04-22T03:03:04.864Z+ grep 'Handoff to Nexus is complete' /var/svc/log/oxide-sled-agent:default.log
11152025-04-22T03:03:04.889Z{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2025-04-22T03:03:04.160763034Z","hostname":"buskin","pid":1567,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1023"}
11162025-04-22T03:03:04.889Z+ echo 'Waited for handoff: 40s'
11172025-04-22T03:03:04.889ZWaited for handoff: 40s
11182025-04-22T03:03:04.889Z+ retry=0
11192025-04-22T03:03:04.889Z++ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i
11202025-04-22T03:03:04.889Z++ wc -l
11212025-04-22T03:03:04.974Znote: database URL not specified. Will search DNS.
11222025-04-22T03:03:04.974Znote: (override with --db-url or OMDB_DB_URL)
11232025-04-22T03:03:05.000Znote: using DNS server for subnet fd00:1122:3344::/48
11242025-04-22T03:03:05.000Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11252025-04-22T03:03:05.000Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11262025-04-22T03:03:05.025Znote: database schema version matches expected (138.0.0)
11272025-04-22T03:03:05.219Z+ ACTUAL_ZPOOL_COUNT=' 9'
11282025-04-22T03:03:05.244Z+ [[ 9 -eq 9 ]]
11292025-04-22T03:03:05.244Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list
11302025-04-22T03:03:05.269Znote: database URL not specified. Will search DNS.
11312025-04-22T03:03:05.269Znote: (override with --db-url or OMDB_DB_URL)
11322025-04-22T03:03:05.269Znote: using DNS server for subnet fd00:1122:3344::/48
11332025-04-22T03:03:05.270Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11342025-04-22T03:03:05.270Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11352025-04-22T03:03:05.270Znote: database schema version matches expected (138.0.0)
11362025-04-22T03:03:05.384ZID |TIME_DELETED |SLED_ID |PHYSICAL_DISK_ID |TOTAL_SIZE |CONTROL_PLANE_STORAGE_BUFFER
11372025-04-22T03:03:05.384Z-------------------------------------+-------------+-------------------------------------+-------------------------------------+------------+-----------------------------
11382025-04-22T03:03:05.384Z0b239f26-3f36-429e-beaa-a90f826d9345 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |cef128ba-a4a6-4edf-99bd-ff4079df1569 |20937965568 |268435456000
11392025-04-22T03:03:05.384Z14a33224-0afc-4c43-ae3d-5fb50dcba3db | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |7d5a4dcb-4620-42b2-bf74-3d264978c08f |20937965568 |268435456000
11402025-04-22T03:03:05.384Z4449a554-c26a-4358-928c-05982e5626b8 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |38bd6f44-aa01-444a-953f-9d2da0b4fc2c |20937965568 |268435456000
11412025-04-22T03:03:05.384Z5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |45bfe972-7c44-4c7b-ba05-4069eca9b337 |20937965568 |268435456000
11422025-04-22T03:03:05.384Z83f4d6e1-915c-4d7f-beea-b28c7a084ba3 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |5f08142f-e34f-4f75-84f7-0a7db7e158df |20937965568 |268435456000
11432025-04-22T03:03:05.384Z8acca6af-d2ba-40aa-a651-5a9c30020d71 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |7fc4d33f-9ece-42b0-a023-b7230996b431 |20937965568 |268435456000
11442025-04-22T03:03:05.384Zbe90f582-5416-4197-b8c2-0bdea6fc95a5 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |e4cdddb8-2485-44d4-bf61-28f702414785 |20937965568 |268435456000
11452025-04-22T03:03:05.384Zce16cc45-6666-4665-b66a-72f8620ea800 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |4b753638-1753-46a4-b9b8-7eb093bf8237 |20937965568 |268435456000
11462025-04-22T03:03:05.384Zddbf788c-925c-40e0-9288-109e29e9e4a9 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |c27cc7f9-df0d-4583-84c5-4893773d5b26 |20937965568 |268435456000
11472025-04-22T03:03:05.409Z++ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i
11482025-04-22T03:03:05.447Znote: database URL not specified. Will search DNS.
11492025-04-22T03:03:05.448Znote: (override with --db-url or OMDB_DB_URL)
11502025-04-22T03:03:05.448Znote: using DNS server for subnet fd00:1122:3344::/48
11512025-04-22T03:03:05.448Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11522025-04-22T03:03:05.448Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11532025-04-22T03:03:05.448Znote: database schema version matches expected (138.0.0)
11542025-04-22T03:03:05.564Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
11552025-04-22T03:03:05.564Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer 0b239f26-3f36-429e-beaa-a90f826d9345 0
11562025-04-22T03:03:05.599Znote: database URL not specified. Will search DNS.
11572025-04-22T03:03:05.599Znote: (override with --db-url or OMDB_DB_URL)
11582025-04-22T03:03:05.599Znote: using DNS server for subnet fd00:1122:3344::/48
11592025-04-22T03:03:05.599Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11602025-04-22T03:03:05.624Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11612025-04-22T03:03:05.624Znote: database schema version matches expected (138.0.0)
11622025-04-22T03:03:05.656Zset pool 0b239f26-3f36-429e-beaa-a90f826d9345 control plane storage buffer bytes to 0
11632025-04-22T03:03:05.681Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
11642025-04-22T03:03:05.681Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer 14a33224-0afc-4c43-ae3d-5fb50dcba3db 0
11652025-04-22T03:03:05.706Znote: database URL not specified. Will search DNS.
11662025-04-22T03:03:05.706Znote: (override with --db-url or OMDB_DB_URL)
11672025-04-22T03:03:05.706Znote: using DNS server for subnet fd00:1122:3344::/48
11682025-04-22T03:03:05.706Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11692025-04-22T03:03:05.706Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11702025-04-22T03:03:05.706Znote: database schema version matches expected (138.0.0)
11712025-04-22T03:03:05.758Zset pool 14a33224-0afc-4c43-ae3d-5fb50dcba3db control plane storage buffer bytes to 0
11722025-04-22T03:03:05.783Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
11732025-04-22T03:03:05.783Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer 4449a554-c26a-4358-928c-05982e5626b8 0
11742025-04-22T03:03:05.808Znote: database URL not specified. Will search DNS.
11752025-04-22T03:03:05.808Znote: (override with --db-url or OMDB_DB_URL)
11762025-04-22T03:03:05.808Znote: using DNS server for subnet fd00:1122:3344::/48
11772025-04-22T03:03:05.808Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11782025-04-22T03:03:05.808Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11792025-04-22T03:03:05.808Znote: database schema version matches expected (138.0.0)
11802025-04-22T03:03:05.858Zset pool 4449a554-c26a-4358-928c-05982e5626b8 control plane storage buffer bytes to 0
11812025-04-22T03:03:05.883Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
11822025-04-22T03:03:05.883Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer 5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc 0
11832025-04-22T03:03:05.908Znote: database URL not specified. Will search DNS.
11842025-04-22T03:03:05.908Znote: (override with --db-url or OMDB_DB_URL)
11852025-04-22T03:03:05.908Znote: using DNS server for subnet fd00:1122:3344::/48
11862025-04-22T03:03:05.908Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11872025-04-22T03:03:05.908Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11882025-04-22T03:03:05.933Znote: database schema version matches expected (138.0.0)
11892025-04-22T03:03:05.958Zset pool 5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc control plane storage buffer bytes to 0
11902025-04-22T03:03:05.983Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
11912025-04-22T03:03:05.983Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer 83f4d6e1-915c-4d7f-beea-b28c7a084ba3 0
11922025-04-22T03:03:06.008Znote: database URL not specified. Will search DNS.
11932025-04-22T03:03:06.008Znote: (override with --db-url or OMDB_DB_URL)
11942025-04-22T03:03:06.008Znote: using DNS server for subnet fd00:1122:3344::/48
11952025-04-22T03:03:06.008Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
11962025-04-22T03:03:06.008Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
11972025-04-22T03:03:06.033Znote: database schema version matches expected (138.0.0)
11982025-04-22T03:03:06.058Zset pool 83f4d6e1-915c-4d7f-beea-b28c7a084ba3 control plane storage buffer bytes to 0
11992025-04-22T03:03:06.083Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
12002025-04-22T03:03:06.083Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer 8acca6af-d2ba-40aa-a651-5a9c30020d71 0
12012025-04-22T03:03:06.108Znote: database URL not specified. Will search DNS.
12022025-04-22T03:03:06.108Znote: (override with --db-url or OMDB_DB_URL)
12032025-04-22T03:03:06.108Znote: using DNS server for subnet fd00:1122:3344::/48
12042025-04-22T03:03:06.108Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
12052025-04-22T03:03:06.108Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
12062025-04-22T03:03:06.133Znote: database schema version matches expected (138.0.0)
12072025-04-22T03:03:06.158Zset pool 8acca6af-d2ba-40aa-a651-5a9c30020d71 control plane storage buffer bytes to 0
12082025-04-22T03:03:06.182Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
12092025-04-22T03:03:06.183Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer be90f582-5416-4197-b8c2-0bdea6fc95a5 0
12102025-04-22T03:03:06.207Znote: database URL not specified. Will search DNS.
12112025-04-22T03:03:06.207Znote: (override with --db-url or OMDB_DB_URL)
12122025-04-22T03:03:06.207Znote: using DNS server for subnet fd00:1122:3344::/48
12132025-04-22T03:03:06.207Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
12142025-04-22T03:03:06.207Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
12152025-04-22T03:03:06.208Znote: database schema version matches expected (138.0.0)
12162025-04-22T03:03:06.258Zset pool be90f582-5416-4197-b8c2-0bdea6fc95a5 control plane storage buffer bytes to 0
12172025-04-22T03:03:06.283Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
12182025-04-22T03:03:06.283Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer ce16cc45-6666-4665-b66a-72f8620ea800 0
12192025-04-22T03:03:06.307Znote: database URL not specified. Will search DNS.
12202025-04-22T03:03:06.307Znote: (override with --db-url or OMDB_DB_URL)
12212025-04-22T03:03:06.307Znote: using DNS server for subnet fd00:1122:3344::/48
12222025-04-22T03:03:06.307Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
12232025-04-22T03:03:06.307Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
12242025-04-22T03:03:06.308Znote: database schema version matches expected (138.0.0)
12252025-04-22T03:03:06.357Zset pool ce16cc45-6666-4665-b66a-72f8620ea800 control plane storage buffer bytes to 0
12262025-04-22T03:03:06.382Z+ for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i)
12272025-04-22T03:03:06.382Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer ddbf788c-925c-40e0-9288-109e29e9e4a9 0
12282025-04-22T03:03:06.407Znote: database URL not specified. Will search DNS.
12292025-04-22T03:03:06.407Znote: (override with --db-url or OMDB_DB_URL)
12302025-04-22T03:03:06.407Znote: using DNS server for subnet fd00:1122:3344::/48
12312025-04-22T03:03:06.407Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
12322025-04-22T03:03:06.407Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
12332025-04-22T03:03:06.407Znote: database schema version matches expected (138.0.0)
12342025-04-22T03:03:06.466Zset pool ddbf788c-925c-40e0-9288-109e29e9e4a9 control plane storage buffer bytes to 0
12352025-04-22T03:03:06.491Z+ pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list
12362025-04-22T03:03:06.516Znote: database URL not specified. Will search DNS.
12372025-04-22T03:03:06.516Znote: (override with --db-url or OMDB_DB_URL)
12382025-04-22T03:03:06.516Znote: using DNS server for subnet fd00:1122:3344::/48
12392025-04-22T03:03:06.516Znote: (if this is not right, use --dns-server to specify an alternate DNS server)
12402025-04-22T03:03:06.516Znote: using database URL postgresql://root@[fd00:1122:3344:101::3]:32221,[fd00:1122:3344:101::6]:32221,[fd00:1122:3344:101::4]:32221,[fd00:1122:3344:101::5]:32221,[fd00:1122:3344:101::7]:32221/omicron?sslmode=disable
12412025-04-22T03:03:06.541Znote: database schema version matches expected (138.0.0)
12422025-04-22T03:03:06.631ZID |TIME_DELETED |SLED_ID |PHYSICAL_DISK_ID |TOTAL_SIZE |CONTROL_PLANE_STORAGE_BUFFER
12432025-04-22T03:03:06.631Z-------------------------------------+-------------+-------------------------------------+-------------------------------------+------------+-----------------------------
12442025-04-22T03:03:06.631Z0b239f26-3f36-429e-beaa-a90f826d9345 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |cef128ba-a4a6-4edf-99bd-ff4079df1569 |20937965568 |0
12452025-04-22T03:03:06.631Z14a33224-0afc-4c43-ae3d-5fb50dcba3db | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |7d5a4dcb-4620-42b2-bf74-3d264978c08f |20937965568 |0
12462025-04-22T03:03:06.631Z4449a554-c26a-4358-928c-05982e5626b8 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |38bd6f44-aa01-444a-953f-9d2da0b4fc2c |20937965568 |0
12472025-04-22T03:03:06.631Z5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |45bfe972-7c44-4c7b-ba05-4069eca9b337 |20937965568 |0
12482025-04-22T03:03:06.631Z83f4d6e1-915c-4d7f-beea-b28c7a084ba3 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |5f08142f-e34f-4f75-84f7-0a7db7e158df |20937965568 |0
12492025-04-22T03:03:06.631Z8acca6af-d2ba-40aa-a651-5a9c30020d71 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |7fc4d33f-9ece-42b0-a023-b7230996b431 |20937965568 |0
12502025-04-22T03:03:06.631Zbe90f582-5416-4197-b8c2-0bdea6fc95a5 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |e4cdddb8-2485-44d4-bf61-28f702414785 |20937965568 |0
12512025-04-22T03:03:06.631Zce16cc45-6666-4665-b66a-72f8620ea800 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |4b753638-1753-46a4-b9b8-7eb093bf8237 |20937965568 |0
12522025-04-22T03:03:06.631Zddbf788c-925c-40e0-9288-109e29e9e4a9 | |cc0806cc-ebea-4a4a-85b3-5b532e0592b4 |c27cc7f9-df0d-4583-84c5-4893773d5b26 |20937965568 |0
12532025-04-22T03:03:06.656Z+ export RUST_BACKTRACE=1
12542025-04-22T03:03:06.656Z+ RUST_BACKTRACE=1
12552025-04-22T03:03:06.656Z+ export E2E_TLS_CERT IPPOOL_START IPPOOL_END
12562025-04-22T03:03:06.656Z++ ./target/debug/bootstrap
12572025-04-22T03:03:07.070ZUsing external DNS server at 10.151.1.164:53
12582025-04-22T03:03:07.097Z2025-04-22 03:03:07.082232865 UTC: attempting to log into API
12592025-04-22T03:03:08.498Z2025-04-22 03:03:08.483446533 UTC: login succeeded
12602025-04-22T03:03:08.531Zwaiting for nexus to come up...
12612025-04-22T03:03:08.642Zcreating IP pool... 10.151.1.175 - 10.151.1.227
12622025-04-22T03:03:09.244Zsetting up quotas...
12632025-04-22T03:03:09.303Zensuring datasets are ready...
12642025-04-22T03:03:13.561Zlet's roll.
12652025-04-22T03:03:13.586Z+ eval 'OXIDE_HOST=https://recovery.sys.oxide.test
12662025-04-22T03:03:13.586ZOXIDE_RESOLVE=recovery.sys.oxide.test:443:10.151.1.167
12672025-04-22T03:03:13.587ZOXIDE_TOKEN=oxide-token-18a0c6cbbebefaf3b613980ac3b4776373e7758d'
12682025-04-22T03:03:13.587Z++ OXIDE_HOST=https://recovery.sys.oxide.test
12692025-04-22T03:03:13.587Z++ OXIDE_RESOLVE=recovery.sys.oxide.test:443:10.151.1.167
12702025-04-22T03:03:13.587Z++ OXIDE_TOKEN=oxide-token-18a0c6cbbebefaf3b613980ac3b4776373e7758d
12712025-04-22T03:03:13.587Z+ export OXIDE_HOST OXIDE_TOKEN
12722025-04-22T03:03:13.587Z+ retry=0
12732025-04-22T03:03:13.587Z+ curl -sSf https://recovery.sys.oxide.test/v1/ping --resolve recovery.sys.oxide.test:443:10.151.1.167 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem
12742025-04-22T03:03:13.652Z+ /usr/oxide/oxide --resolve recovery.sys.oxide.test:443:10.151.1.167 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem project create --name images --description 'some images'
12752025-04-22T03:03:14.489Z{"status":"ok"}{
12762025-04-22T03:03:14.489Z "description": "some images",
12772025-04-22T03:03:14.489Z "id": "559098cd-969a-4715-836c-4cf874b9484f",
12782025-04-22T03:03:14.489Z "name": "images",
12792025-04-22T03:03:14.489Z "time_created": "2025-04-22T03:03:13.745385Z",
12802025-04-22T03:03:14.489Z "time_modified": "2025-04-22T03:03:13.745385Z"
12812025-04-22T03:03:14.489Z}
12822025-04-22T03:03:14.515Z+ /usr/oxide/oxide --resolve recovery.sys.oxide.test:443:10.151.1.167 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem disk import --path debian-11-genericcloud-amd64.raw --disk debian11-boot --project images --description 'debian 11 cloud image from distros' --snapshot debian11-snapshot --image debian11 --image-description 'debian 11 original base image' --image-os debian --image-version 11 --parallelism 1
 
12832025-04-22T03:05:30.704ZApr 22 03:05:30 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:99.28.14.242:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
12842025-04-22T03:09:31.121ZApr 22 03:09:30 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:129.250.35.250:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
12852025-04-22T03:10:14.821ZApr 22 03:10:14 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:104.131.139.195:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
12862025-04-22T03:11:03.012ZApr 22 03:11:02 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:104.167.241.253:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
12872025-04-22T03:11:17.629ZApr 22 03:11:17 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:23.155.40.38:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
12882025-04-22T03:12:13.943ZApr 22 03:12:13 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:104.167.241.253:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
 
12892025-04-22T03:13:52.313ZDone!
12902025-04-22T03:13:52.338Z+ /usr/oxide/oxide --resolve recovery.sys.oxide.test:443:10.151.1.167 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem image promote --project images --image debian11
12912025-04-22T03:13:52.454Z{
12922025-04-22T03:13:52.454Z "block_size": 512,
12932025-04-22T03:13:52.454Z "description": "debian 11 original base image",
12942025-04-22T03:13:52.454Z "id": "81d193b4-e07b-4693-b7db-c521b5db3724",
12952025-04-22T03:13:52.454Z "name": "debian11",
12962025-04-22T03:13:52.455Z "os": "debian",
12972025-04-22T03:13:52.455Z "size": 2147483648,
12982025-04-22T03:13:52.455Z "time_created": "2025-04-22T03:13:52.186238Z",
12992025-04-22T03:13:52.455Z "time_modified": "2025-04-22T03:13:52.427821Z",
13002025-04-22T03:13:52.455Z "version": "11"
13012025-04-22T03:13:52.455Z}
13022025-04-22T03:13:52.480Z+ for test_bin in tests/*
13032025-04-22T03:13:52.480Z+ ./tests/bootstrap-c6b0d27f0cb47d17
13042025-04-22T03:13:52.518Z
13052025-04-22T03:13:52.518Zrunning 0 tests
13062025-04-22T03:13:52.543Z
13072025-04-22T03:13:52.543Ztest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s
13082025-04-22T03:13:52.543Z
13092025-04-22T03:13:52.543Z+ for test_bin in tests/*
13102025-04-22T03:13:52.543Z+ ./tests/commtest-38aab2992c80b05a
13112025-04-22T03:13:52.569Z
13122025-04-22T03:13:52.569Zrunning 0 tests
13132025-04-22T03:13:52.595Z
13142025-04-22T03:13:52.595Ztest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s
13152025-04-22T03:13:52.595Z
13162025-04-22T03:13:52.595Z+ for test_bin in tests/*
13172025-04-22T03:13:52.595Z+ ./tests/dhcp_server-1c4aecca2e768282
13182025-04-22T03:13:52.620Z
13192025-04-22T03:13:52.620Zrunning 0 tests
13202025-04-22T03:13:52.620Z
13212025-04-22T03:13:52.620Ztest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s
13222025-04-22T03:13:52.620Z
13232025-04-22T03:13:52.620Z+ for test_bin in tests/*
13242025-04-22T03:13:52.620Z+ ./tests/end_to_end_tests-804095052923f3b9
13252025-04-22T03:13:53.028Z
13262025-04-22T03:13:53.028Zrunning 3 tests
13272025-04-22T03:13:53.067Znote: configured to log to "/tmp/end_to_end_tests-804095052923f3b9-new_blueprint_noop.1222.0.log"
13282025-04-22T03:13:53.444Ztest no_spoof::no_spoof_here ... ok
13292025-04-22T03:13:54.992Ztest noop_blueprint::new_blueprint_noop ... ok
13302025-04-22T03:14:22.291Ztest instance_launch::instance_launch ... ok
13312025-04-22T03:14:22.291Z
13322025-04-22T03:14:22.316Ztest result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 29.26s
13332025-04-22T03:14:22.316Z
13342025-04-22T03:14:22.316Z+ _exit_trap
13352025-04-22T03:14:22.316Z+ local status=0
13362025-04-22T03:14:22.316Z+ set +o errexit
13372025-04-22T03:14:22.316Z+ [[ x != \x ]]
13382025-04-22T03:14:22.316Z+ pfexec svcadm -Z disable -s cron
13392025-04-22T03:14:24.743Z+ [[ 0 -eq 0 ]]
13402025-04-22T03:14:24.743Z+ exit 0
13412025-04-22T03:14:24.768Zprocess exited: duration 899834 ms, exit code 0
 
13422025-04-22T03:14:24.914Zfound 110 output files
13432025-04-22T03:14:24.967Zuploading: /var/svc/log/oxide-mg-ddm:default.log (314963 bytes)
13442025-04-22T03:14:26.229Zuploaded: /var/svc/log/oxide-mg-ddm:default.log
13452025-04-22T03:14:26.229Zuploading: /var/svc/log/oxide-sled-agent:default.log (226918 bytes)
13462025-04-22T03:14:27.494Zuploaded: /var/svc/log/oxide-sled-agent:default.log
13472025-04-22T03:14:27.495Zuploading: /var/svc/log/oxide-sled-agent:default.log.0 (4073879 bytes)
13482025-04-22T03:14:30.577Zuploaded: /var/svc/log/oxide-sled-agent:default.log.0
13492025-04-22T03:14:30.578Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-dendrite:default.log (7768849 bytes)
13502025-04-22T03:14:33.812Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-dendrite:default.log" changed size mid upload: 7768849 -> 7871756
13512025-04-22T03:14:34.883Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-dendrite:default.log
13522025-04-22T03:14:34.883Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-lldpd:default.log (39230 bytes)
13532025-04-22T03:14:35.020Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-lldpd:default.log" changed size mid upload: 39230 -> 39527
13542025-04-22T03:14:36.092Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-lldpd:default.log
13552025-04-22T03:14:36.116Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-mg-ddm:default.log (347818 bytes)
13562025-04-22T03:14:36.698Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-mg-ddm:default.log" changed size mid upload: 347818 -> 355052
13572025-04-22T03:14:37.772Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-mg-ddm:default.log
13582025-04-22T03:14:37.821Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-mgd:default.log (439419 bytes)
13592025-04-22T03:14:38.466Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-mgd:default.log" changed size mid upload: 439419 -> 451433
13602025-04-22T03:14:39.548Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-mgd:default.log
13612025-04-22T03:14:39.548Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-mgs:default.log (2235216 bytes)
13622025-04-22T03:14:41.255Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-mgs:default.log" changed size mid upload: 2235216 -> 2281090
13632025-04-22T03:14:42.327Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-mgs:default.log
13642025-04-22T03:14:42.327Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-sp-sim:default.log (5439078 bytes)
13652025-04-22T03:14:45.025Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-sp-sim:default.log" changed size mid upload: 5439078 -> 5568960
13662025-04-22T03:14:46.098Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-sp-sim:default.log
13672025-04-22T03:14:46.098Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-switch_zone_setup:default.log (3338 bytes)
13682025-04-22T03:14:47.194Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-switch_zone_setup:default.log
13692025-04-22T03:14:47.194Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-tfport:default.log (486630 bytes)
13702025-04-22T03:14:47.955Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-tfport:default.log" changed size mid upload: 486630 -> 500010
13712025-04-22T03:14:49.027Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-tfport:default.log
13722025-04-22T03:14:49.027Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-uplink:default.log (13705 bytes)
13732025-04-22T03:14:49.126Zupload warning: file "/zone/oxz_switch/root/var/svc/log/oxide-uplink:default.log" changed size mid upload: 13705 -> 14290
13742025-04-22T03:14:50.198Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-uplink:default.log
13752025-04-22T03:14:50.198Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-wicketd:default.log (4216 bytes)
13762025-04-22T03:14:51.293Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-wicketd:default.log
13772025-04-22T03:14:51.293Zuploading: /zone/oxz_switch/root/var/svc/log/oxide-zone-network-setup:default.log (1662 bytes)
13782025-04-22T03:14:52.393Zuploaded: /zone/oxz_switch/root/var/svc/log/oxide-zone-network-setup:default.log
13792025-04-22T03:14:52.393Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-agent:default.log (20603 bytes)
13802025-04-22T03:14:53.535Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-agent:default.log
13812025-04-22T03:14:53.535Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
13822025-04-22T03:14:54.632Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-downstairs:default.log
13832025-04-22T03:14:54.657Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-downstairs:downstairs-c7dfc4d6-9a43-4bc4-a0cd-0636111ba1af.log (6080 bytes)
13842025-04-22T03:14:55.728Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-downstairs:downstairs-c7dfc4d6-9a43-4bc4-a0cd-0636111ba1af.log
13852025-04-22T03:14:55.728Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-downstairs:snapshot-c7dfc4d6-9a43-4bc4-a0cd-0636111ba1af-215b9cb3-2af4-4667-a31f-61646d6a4191.log (5656 bytes)
13862025-04-22T03:14:56.829Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-crucible-downstairs:snapshot-c7dfc4d6-9a43-4bc4-a0cd-0636111ba1af-215b9cb3-2af4-4667-a31f-61646d6a4191.log
13872025-04-22T03:14:56.829Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-zone-network-setup:default.log (2083 bytes)
13882025-04-22T03:14:57.932Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_crucible_60fddec4-f5fc-4e72-bd91-30e886bd157a/root/var/svc/log/oxide-zone-network-setup:default.log
13892025-04-22T03:14:57.981Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-nexus:default.log (42105811 bytes)
13902025-04-22T03:15:06.359Zupload warning: file "/pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 42105811 -> 42526012
13912025-04-22T03:15:07.444Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-nexus:default.log
13922025-04-22T03:15:07.444Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-opte-interface-setup:default.log (1134 bytes)
13932025-04-22T03:15:08.535Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-opte-interface-setup:default.log
13942025-04-22T03:15:08.536Zuploading: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-zone-network-setup:default.log (2057 bytes)
13952025-04-22T03:15:09.660Zuploaded: /pool/ext/0b239f26-3f36-429e-beaa-a90f826d9345/crypt/zone/oxz_nexus_28f98df8-c5f7-422f-a604-52a74d7a283c/root/var/svc/log/oxide-zone-network-setup:default.log
13962025-04-22T03:15:09.661Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_cockroachdb_7bc94dc9-929c-4021-a5fd-a8df5639da67/root/var/svc/log/oxide-cockroach-admin:default.log (4706 bytes)
13972025-04-22T03:15:10.832Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_cockroachdb_7bc94dc9-929c-4021-a5fd-a8df5639da67/root/var/svc/log/oxide-cockroach-admin:default.log
13982025-04-22T03:15:10.832Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_cockroachdb_7bc94dc9-929c-4021-a5fd-a8df5639da67/root/var/svc/log/oxide-cockroachdb:default.log (4326 bytes)
13992025-04-22T03:15:12.409Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_cockroachdb_7bc94dc9-929c-4021-a5fd-a8df5639da67/root/var/svc/log/oxide-cockroachdb:default.log
14002025-04-22T03:15:12.435Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_cockroachdb_7bc94dc9-929c-4021-a5fd-a8df5639da67/root/var/svc/log/oxide-zone-network-setup:default.log (2101 bytes)
14012025-04-22T03:15:13.522Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_cockroachdb_7bc94dc9-929c-4021-a5fd-a8df5639da67/root/var/svc/log/oxide-zone-network-setup:default.log
14022025-04-22T03:15:13.526Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-agent:default.log (18998 bytes)
14032025-04-22T03:15:14.704Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-agent:default.log
14042025-04-22T03:15:14.730Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
14052025-04-22T03:15:15.806Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-downstairs:default.log
14062025-04-22T03:15:15.806Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-downstairs:downstairs-40b2c042-5a5f-418f-9ab1-b971b28b6ead.log (3125 bytes)
14072025-04-22T03:15:16.903Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-downstairs:downstairs-40b2c042-5a5f-418f-9ab1-b971b28b6ead.log
14082025-04-22T03:15:16.928Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-downstairs:downstairs-b8d02697-3794-49e8-8135-724a1e333765.log (3009 bytes)
14092025-04-22T03:15:18.000Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-crucible-downstairs:downstairs-b8d02697-3794-49e8-8135-724a1e333765.log
14102025-04-22T03:15:18.000Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-zone-network-setup:default.log (2081 bytes)
14112025-04-22T03:15:19.105Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_da925ad5-0394-46ad-96fa-8fe9b34a0cde/root/var/svc/log/oxide-zone-network-setup:default.log
14122025-04-22T03:15:19.105Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_pantry_4ee4cb5f-e162-497e-bfdc-706eff8c3769/root/var/svc/log/oxide-crucible-pantry:default.log (922949 bytes)
14132025-04-22T03:15:20.227Zupload warning: file "/pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_pantry_4ee4cb5f-e162-497e-bfdc-706eff8c3769/root/var/svc/log/oxide-crucible-pantry:default.log" changed size mid upload: 922949 -> 929642
14142025-04-22T03:15:21.300Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_pantry_4ee4cb5f-e162-497e-bfdc-706eff8c3769/root/var/svc/log/oxide-crucible-pantry:default.log
14152025-04-22T03:15:21.300Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_pantry_4ee4cb5f-e162-497e-bfdc-706eff8c3769/root/var/svc/log/oxide-zone-network-setup:default.log (2139 bytes)
14162025-04-22T03:15:22.411Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_crucible_pantry_4ee4cb5f-e162-497e-bfdc-706eff8c3769/root/var/svc/log/oxide-zone-network-setup:default.log
14172025-04-22T03:15:22.478Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-nexus:default.log (4895618 bytes)
14182025-04-22T03:15:25.260Zupload warning: file "/pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 4895618 -> 5260722
14192025-04-22T03:15:26.334Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-nexus:default.log
14202025-04-22T03:15:26.334Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-opte-interface-setup:default.log (1133 bytes)
14212025-04-22T03:15:27.432Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-opte-interface-setup:default.log
14222025-04-22T03:15:27.456Zuploading: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-zone-network-setup:default.log (2056 bytes)
14232025-04-22T03:15:28.528Zuploaded: /pool/ext/14a33224-0afc-4c43-ae3d-5fb50dcba3db/crypt/zone/oxz_nexus_b67b4ae7-b464-4d25-a198-9d78a495ed77/root/var/svc/log/oxide-zone-network-setup:default.log
14242025-04-22T03:15:28.553Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-clickhouse-admin-single:default.log (936 bytes)
14252025-04-22T03:15:29.625Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-clickhouse-admin-single:default.log
14262025-04-22T03:15:29.625Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-clickhouse:default.log (3715427 bytes)
14272025-04-22T03:15:32.024Zupload warning: file "/pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-clickhouse:default.log" changed size mid upload: 3715427 -> 4074844
14282025-04-22T03:15:33.095Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-clickhouse:default.log
14292025-04-22T03:15:33.095Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-zone-network-setup:default.log (2096 bytes)
14302025-04-22T03:15:34.192Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_clickhouse_2f8498fd-9869-481f-a508-8ce12d4c9d61/root/var/svc/log/oxide-zone-network-setup:default.log
14312025-04-22T03:15:34.192Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_cockroachdb_2cea94cc-3dee-4293-a0d3-075142a08e32/root/var/svc/log/oxide-cockroach-admin:default.log (9201 bytes)
14322025-04-22T03:15:35.310Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_cockroachdb_2cea94cc-3dee-4293-a0d3-075142a08e32/root/var/svc/log/oxide-cockroach-admin:default.log
14332025-04-22T03:15:35.310Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_cockroachdb_2cea94cc-3dee-4293-a0d3-075142a08e32/root/var/svc/log/oxide-cockroachdb:default.log (4300 bytes)
14342025-04-22T03:15:36.408Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_cockroachdb_2cea94cc-3dee-4293-a0d3-075142a08e32/root/var/svc/log/oxide-cockroachdb:default.log
14352025-04-22T03:15:36.439Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_cockroachdb_2cea94cc-3dee-4293-a0d3-075142a08e32/root/var/svc/log/oxide-zone-network-setup:default.log (2101 bytes)
14362025-04-22T03:15:37.512Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_cockroachdb_2cea94cc-3dee-4293-a0d3-075142a08e32/root/var/svc/log/oxide-zone-network-setup:default.log
14372025-04-22T03:15:37.512Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_crucible_d2cfb035-c90b-4e0e-b1a7-f00718944176/root/var/svc/log/oxide-crucible-agent:default.log (1555 bytes)
14382025-04-22T03:15:38.608Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_crucible_d2cfb035-c90b-4e0e-b1a7-f00718944176/root/var/svc/log/oxide-crucible-agent:default.log
14392025-04-22T03:15:38.632Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_crucible_d2cfb035-c90b-4e0e-b1a7-f00718944176/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
14402025-04-22T03:15:39.704Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_crucible_d2cfb035-c90b-4e0e-b1a7-f00718944176/root/var/svc/log/oxide-crucible-downstairs:default.log
14412025-04-22T03:15:39.704Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_crucible_d2cfb035-c90b-4e0e-b1a7-f00718944176/root/var/svc/log/oxide-zone-network-setup:default.log (2083 bytes)
14422025-04-22T03:15:40.799Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_crucible_d2cfb035-c90b-4e0e-b1a7-f00718944176/root/var/svc/log/oxide-zone-network-setup:default.log
14432025-04-22T03:15:40.799Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-external_dns:default.log (45791 bytes)
14442025-04-22T03:15:41.002Zupload warning: file "/pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-external_dns:default.log" changed size mid upload: 45791 -> 49568
14452025-04-22T03:15:42.075Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-external_dns:default.log
14462025-04-22T03:15:42.075Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-opte-interface-setup:default.log (1156 bytes)
14472025-04-22T03:15:43.172Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-opte-interface-setup:default.log
14482025-04-22T03:15:43.196Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-zone-network-setup:default.log (2113 bytes)
14492025-04-22T03:15:44.267Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_external_dns_81774d13-0e99-4dc9-abdf-daef2a6cb0b0/root/var/svc/log/oxide-zone-network-setup:default.log
14502025-04-22T03:15:44.267Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_internal_dns_c2f4433e-4f54-4a7a-8462-24d94a6b64d4/root/var/svc/log/oxide-internal_dns:default.log (679538 bytes)
14512025-04-22T03:15:45.234Zupload warning: file "/pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_internal_dns_c2f4433e-4f54-4a7a-8462-24d94a6b64d4/root/var/svc/log/oxide-internal_dns:default.log" changed size mid upload: 679538 -> 705818
14522025-04-22T03:15:46.307Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_internal_dns_c2f4433e-4f54-4a7a-8462-24d94a6b64d4/root/var/svc/log/oxide-internal_dns:default.log
14532025-04-22T03:15:46.331Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_internal_dns_c2f4433e-4f54-4a7a-8462-24d94a6b64d4/root/var/svc/log/oxide-zone-network-setup:default.log (2092 bytes)
14542025-04-22T03:15:47.402Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_internal_dns_c2f4433e-4f54-4a7a-8462-24d94a6b64d4/root/var/svc/log/oxide-zone-network-setup:default.log
14552025-04-22T03:15:47.402Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_oximeter_bf12c33c-45c7-472d-9735-f52f6f5b0936/root/var/svc/log/oxide-oximeter:default.log (3825327 bytes)
14562025-04-22T03:15:49.890Zupload warning: file "/pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_oximeter_bf12c33c-45c7-472d-9735-f52f6f5b0936/root/var/svc/log/oxide-oximeter:default.log" changed size mid upload: 3825327 -> 4320521
14572025-04-22T03:15:50.964Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_oximeter_bf12c33c-45c7-472d-9735-f52f6f5b0936/root/var/svc/log/oxide-oximeter:default.log
14582025-04-22T03:15:50.988Zuploading: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_oximeter_bf12c33c-45c7-472d-9735-f52f6f5b0936/root/var/svc/log/oxide-zone-network-setup:default.log (2079 bytes)
14592025-04-22T03:15:51.630ZApr 22 03:15:51 buskin xde: WARNING: failed to generate descriptor for stateful action: ICMP:172.30.3.5:0:72.30.35.88:0 Unexpected { msg: "Expected ICMP Echo Request, found: 3" }
14602025-04-22T03:15:52.061Zuploaded: /pool/ext/4449a554-c26a-4358-928c-05982e5626b8/crypt/zone/oxz_oximeter_bf12c33c-45c7-472d-9735-f52f6f5b0936/root/var/svc/log/oxide-zone-network-setup:default.log
14612025-04-22T03:15:52.061Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_cockroachdb_4812a882-3d2d-4068-8af9-31850db773c6/root/var/svc/log/oxide-cockroach-admin:default.log (4705 bytes)
14622025-04-22T03:15:53.158Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_cockroachdb_4812a882-3d2d-4068-8af9-31850db773c6/root/var/svc/log/oxide-cockroach-admin:default.log
14632025-04-22T03:15:53.158Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_cockroachdb_4812a882-3d2d-4068-8af9-31850db773c6/root/var/svc/log/oxide-cockroachdb:default.log (4326 bytes)
14642025-04-22T03:15:54.256Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_cockroachdb_4812a882-3d2d-4068-8af9-31850db773c6/root/var/svc/log/oxide-cockroachdb:default.log
14652025-04-22T03:15:54.256Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_cockroachdb_4812a882-3d2d-4068-8af9-31850db773c6/root/var/svc/log/oxide-zone-network-setup:default.log (2093 bytes)
14662025-04-22T03:15:55.352Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_cockroachdb_4812a882-3d2d-4068-8af9-31850db773c6/root/var/svc/log/oxide-zone-network-setup:default.log
14672025-04-22T03:15:55.377Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-agent:default.log (19001 bytes)
14682025-04-22T03:15:56.516Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-agent:default.log
14692025-04-22T03:15:56.516Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
14702025-04-22T03:15:57.613Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-downstairs:default.log
14712025-04-22T03:15:57.638Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-downstairs:downstairs-7cd5eb66-301e-4011-ba35-9adece74fa40.log (3125 bytes)
14722025-04-22T03:15:58.710Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-downstairs:downstairs-7cd5eb66-301e-4011-ba35-9adece74fa40.log
14732025-04-22T03:15:58.710Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-downstairs:downstairs-7dac7392-459c-42cf-9223-2a978db8dcae.log (3009 bytes)
14742025-04-22T03:15:59.806Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-crucible-downstairs:downstairs-7dac7392-459c-42cf-9223-2a978db8dcae.log
14752025-04-22T03:15:59.806Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-zone-network-setup:default.log (2082 bytes)
14762025-04-22T03:16:00.902Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_crucible_e3f8b0b9-939f-4989-85a2-78770d2dc797/root/var/svc/log/oxide-zone-network-setup:default.log
14772025-04-22T03:16:00.902Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-external_dns:default.log (45802 bytes)
14782025-04-22T03:16:01.107Zupload warning: file "/pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-external_dns:default.log" changed size mid upload: 45802 -> 53353
14792025-04-22T03:16:02.179Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-external_dns:default.log
14802025-04-22T03:16:02.179Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-opte-interface-setup:default.log (1156 bytes)
14812025-04-22T03:16:03.276Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-opte-interface-setup:default.log
14822025-04-22T03:16:03.277Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-zone-network-setup:default.log (2113 bytes)
14832025-04-22T03:16:04.372Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_external_dns_87f521a5-be93-414d-b7e3-0651c3a5831c/root/var/svc/log/oxide-zone-network-setup:default.log
14842025-04-22T03:16:04.400Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_internal_dns_a801ea93-43c6-4996-a1dc-3615fc47d8aa/root/var/svc/log/oxide-internal_dns:default.log (679526 bytes)
14852025-04-22T03:16:05.382Zupload warning: file "/pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_internal_dns_a801ea93-43c6-4996-a1dc-3615fc47d8aa/root/var/svc/log/oxide-internal_dns:default.log" changed size mid upload: 679526 -> 718010
14862025-04-22T03:16:06.458Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_internal_dns_a801ea93-43c6-4996-a1dc-3615fc47d8aa/root/var/svc/log/oxide-internal_dns:default.log
14872025-04-22T03:16:06.458Zuploading: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_internal_dns_a801ea93-43c6-4996-a1dc-3615fc47d8aa/root/var/svc/log/oxide-zone-network-setup:default.log (2093 bytes)
14882025-04-22T03:16:07.555Zuploaded: /pool/ext/5eb4fc2a-20cf-4c47-b5b7-62ac2dc7aabc/crypt/zone/oxz_internal_dns_a801ea93-43c6-4996-a1dc-3615fc47d8aa/root/var/svc/log/oxide-zone-network-setup:default.log
14892025-04-22T03:16:07.555Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_cockroachdb_3b44dc6f-dcfd-42e2-8710-77936959bf33/root/var/svc/log/oxide-cockroach-admin:default.log (4707 bytes)
14902025-04-22T03:16:08.654Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_cockroachdb_3b44dc6f-dcfd-42e2-8710-77936959bf33/root/var/svc/log/oxide-cockroach-admin:default.log
14912025-04-22T03:16:08.680Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_cockroachdb_3b44dc6f-dcfd-42e2-8710-77936959bf33/root/var/svc/log/oxide-cockroachdb:default.log (4326 bytes)
14922025-04-22T03:16:09.753Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_cockroachdb_3b44dc6f-dcfd-42e2-8710-77936959bf33/root/var/svc/log/oxide-cockroachdb:default.log
14932025-04-22T03:16:09.753Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_cockroachdb_3b44dc6f-dcfd-42e2-8710-77936959bf33/root/var/svc/log/oxide-zone-network-setup:default.log (2100 bytes)
14942025-04-22T03:16:10.849Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_cockroachdb_3b44dc6f-dcfd-42e2-8710-77936959bf33/root/var/svc/log/oxide-zone-network-setup:default.log
14952025-04-22T03:16:10.849Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_514733ac-dc82-431a-bfbf-30406fab5087/root/var/svc/log/oxide-crucible-agent:default.log (1556 bytes)
14962025-04-22T03:16:11.945Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_514733ac-dc82-431a-bfbf-30406fab5087/root/var/svc/log/oxide-crucible-agent:default.log
14972025-04-22T03:16:11.945Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_514733ac-dc82-431a-bfbf-30406fab5087/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
14982025-04-22T03:16:13.043Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_514733ac-dc82-431a-bfbf-30406fab5087/root/var/svc/log/oxide-crucible-downstairs:default.log
14992025-04-22T03:16:13.068Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_514733ac-dc82-431a-bfbf-30406fab5087/root/var/svc/log/oxide-zone-network-setup:default.log (2082 bytes)
15002025-04-22T03:16:14.141Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_514733ac-dc82-431a-bfbf-30406fab5087/root/var/svc/log/oxide-zone-network-setup:default.log
15012025-04-22T03:16:14.141Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_pantry_a9135f26-af8c-4f07-ae11-421262931b80/root/var/svc/log/oxide-crucible-pantry:default.log (161432 bytes)
15022025-04-22T03:16:14.326Zupload warning: file "/pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_pantry_a9135f26-af8c-4f07-ae11-421262931b80/root/var/svc/log/oxide-crucible-pantry:default.log" changed size mid upload: 161432 -> 181203
15032025-04-22T03:16:15.398Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_pantry_a9135f26-af8c-4f07-ae11-421262931b80/root/var/svc/log/oxide-crucible-pantry:default.log
15042025-04-22T03:16:15.446Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_pantry_a9135f26-af8c-4f07-ae11-421262931b80/root/var/svc/log/oxide-zone-network-setup:default.log (2137 bytes)
15052025-04-22T03:16:16.497Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_crucible_pantry_a9135f26-af8c-4f07-ae11-421262931b80/root/var/svc/log/oxide-zone-network-setup:default.log
15062025-04-22T03:16:16.497Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-chrony-setup:default.log (15276 bytes)
15072025-04-22T03:16:17.620Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-chrony-setup:default.log
15082025-04-22T03:16:17.646Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-ntp:default.log (2160 bytes)
15092025-04-22T03:16:18.720Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-ntp:default.log
15102025-04-22T03:16:18.720Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-opte-interface-setup:default.log (1125 bytes)
15112025-04-22T03:16:19.819Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-opte-interface-setup:default.log
15122025-04-22T03:16:19.844Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-zone-network-setup:default.log (2032 bytes)
15132025-04-22T03:16:20.918Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/svc/log/oxide-zone-network-setup:default.log
15142025-04-22T03:16:20.918Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-agent:default.log (36031 bytes)
15152025-04-22T03:16:22.063Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-agent:default.log
15162025-04-22T03:16:22.064Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
15172025-04-22T03:16:23.385Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:default.log
15182025-04-22T03:16:23.415Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:downstairs-3d7f04b0-295c-4848-9921-e70761ff24ef.log (5974 bytes)
15192025-04-22T03:16:24.491Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:downstairs-3d7f04b0-295c-4848-9921-e70761ff24ef.log
15202025-04-22T03:16:24.491Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3.log (6078 bytes)
15212025-04-22T03:16:25.590Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3.log
15222025-04-22T03:16:25.590Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:snapshot-4570144a-fdc4-4f97-ac49-4899197679e3-215b9cb3-2af4-4667-a31f-61646d6a4191.log (5656 bytes)
15232025-04-22T03:16:26.688Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-crucible-downstairs:snapshot-4570144a-fdc4-4f97-ac49-4899197679e3-215b9cb3-2af4-4667-a31f-61646d6a4191.log
15242025-04-22T03:16:26.688Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-zone-network-setup:default.log (2082 bytes)
15252025-04-22T03:16:27.786Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c/root/var/svc/log/oxide-zone-network-setup:default.log
15262025-04-22T03:16:27.786Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-nexus:default.log (4921145 bytes)
15272025-04-22T03:16:28.256Zupload warning: file "/pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 4921145 -> 5743081
15282025-04-22T03:16:29.330Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-nexus:default.log
15292025-04-22T03:16:29.330Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-opte-interface-setup:default.log (1135 bytes)
15302025-04-22T03:16:30.469Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-opte-interface-setup:default.log
15312025-04-22T03:16:30.469Zuploading: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-zone-network-setup:default.log (2054 bytes)
15322025-04-22T03:16:31.543Zuploaded: /pool/ext/8acca6af-d2ba-40aa-a651-5a9c30020d71/crypt/zone/oxz_nexus_d5b0844c-9efb-49a6-b52e-7a812df5bb5e/root/var/svc/log/oxide-zone-network-setup:default.log
15332025-04-22T03:16:31.543Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-agent:default.log (25320 bytes)
15342025-04-22T03:16:32.688Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-agent:default.log
15352025-04-22T03:16:32.713Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
15362025-04-22T03:16:33.788Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-downstairs:default.log
15372025-04-22T03:16:33.788Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-downstairs:downstairs-765afa79-78b9-45d9-ad63-3be2752d64f8.log (5970 bytes)
15382025-04-22T03:16:34.887Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-downstairs:downstairs-765afa79-78b9-45d9-ad63-3be2752d64f8.log
15392025-04-22T03:16:34.887Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-downstairs:downstairs-f881f330-323d-4fcd-abb2-d7dc3c346157.log (3124 bytes)
15402025-04-22T03:16:35.985Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-crucible-downstairs:downstairs-f881f330-323d-4fcd-abb2-d7dc3c346157.log
15412025-04-22T03:16:35.985Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-zone-network-setup:default.log (2082 bytes)
15422025-04-22T03:16:37.084Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_2106e2c1-f750-4de6-aff4-b0544a00ebc1/root/var/svc/log/oxide-zone-network-setup:default.log
15432025-04-22T03:16:37.084Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_pantry_5ea7d875-51cf-4a81-80a7-5f3140631e7c/root/var/svc/log/oxide-crucible-pantry:default.log (160996 bytes)
15442025-04-22T03:16:37.273Zupload warning: file "/pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_pantry_5ea7d875-51cf-4a81-80a7-5f3140631e7c/root/var/svc/log/oxide-crucible-pantry:default.log" changed size mid upload: 160996 -> 187695
15452025-04-22T03:16:38.348Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_pantry_5ea7d875-51cf-4a81-80a7-5f3140631e7c/root/var/svc/log/oxide-crucible-pantry:default.log
15462025-04-22T03:16:38.348Zuploading: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_pantry_5ea7d875-51cf-4a81-80a7-5f3140631e7c/root/var/svc/log/oxide-zone-network-setup:default.log (2137 bytes)
15472025-04-22T03:16:39.445Zuploaded: /pool/ext/be90f582-5416-4197-b8c2-0bdea6fc95a5/crypt/zone/oxz_crucible_pantry_5ea7d875-51cf-4a81-80a7-5f3140631e7c/root/var/svc/log/oxide-zone-network-setup:default.log
15482025-04-22T03:16:39.445Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_cockroachdb_4ac2bf71-139c-4397-b14d-99b6aeedf342/root/var/svc/log/oxide-cockroach-admin:default.log (4703 bytes)
15492025-04-22T03:16:40.544Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_cockroachdb_4ac2bf71-139c-4397-b14d-99b6aeedf342/root/var/svc/log/oxide-cockroach-admin:default.log
15502025-04-22T03:16:40.544Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_cockroachdb_4ac2bf71-139c-4397-b14d-99b6aeedf342/root/var/svc/log/oxide-cockroachdb:default.log (4326 bytes)
15512025-04-22T03:16:41.647Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_cockroachdb_4ac2bf71-139c-4397-b14d-99b6aeedf342/root/var/svc/log/oxide-cockroachdb:default.log
15522025-04-22T03:16:41.648Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_cockroachdb_4ac2bf71-139c-4397-b14d-99b6aeedf342/root/var/svc/log/oxide-zone-network-setup:default.log (2100 bytes)
15532025-04-22T03:16:42.749Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_cockroachdb_4ac2bf71-139c-4397-b14d-99b6aeedf342/root/var/svc/log/oxide-zone-network-setup:default.log
15542025-04-22T03:16:42.749Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_crucible_d8f094ea-0005-4230-9bfa-840e8a64c2f8/root/var/svc/log/oxide-crucible-agent:default.log (1552 bytes)
15552025-04-22T03:16:43.874Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_crucible_d8f094ea-0005-4230-9bfa-840e8a64c2f8/root/var/svc/log/oxide-crucible-agent:default.log
15562025-04-22T03:16:43.875Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_crucible_d8f094ea-0005-4230-9bfa-840e8a64c2f8/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
15572025-04-22T03:16:44.972Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_crucible_d8f094ea-0005-4230-9bfa-840e8a64c2f8/root/var/svc/log/oxide-crucible-downstairs:default.log
15582025-04-22T03:16:44.997Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_crucible_d8f094ea-0005-4230-9bfa-840e8a64c2f8/root/var/svc/log/oxide-zone-network-setup:default.log (2081 bytes)
15592025-04-22T03:16:46.072Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_crucible_d8f094ea-0005-4230-9bfa-840e8a64c2f8/root/var/svc/log/oxide-zone-network-setup:default.log
15602025-04-22T03:16:46.072Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_internal_dns_77b0b6c4-275b-4a59-874e-b7e6c8b247ad/root/var/svc/log/oxide-internal_dns:default.log (681943 bytes)
15612025-04-22T03:16:46.353Zupload warning: file "/pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_internal_dns_77b0b6c4-275b-4a59-874e-b7e6c8b247ad/root/var/svc/log/oxide-internal_dns:default.log" changed size mid upload: 681943 -> 731691
15622025-04-22T03:16:47.428Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_internal_dns_77b0b6c4-275b-4a59-874e-b7e6c8b247ad/root/var/svc/log/oxide-internal_dns:default.log
15632025-04-22T03:16:47.453Zuploading: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_internal_dns_77b0b6c4-275b-4a59-874e-b7e6c8b247ad/root/var/svc/log/oxide-zone-network-setup:default.log (2094 bytes)
15642025-04-22T03:16:48.525Zuploaded: /pool/ext/ce16cc45-6666-4665-b66a-72f8620ea800/crypt/zone/oxz_internal_dns_77b0b6c4-275b-4a59-874e-b7e6c8b247ad/root/var/svc/log/oxide-zone-network-setup:default.log
15652025-04-22T03:16:48.525Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-agent:default.log (44825 bytes)
15662025-04-22T03:16:49.692Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-agent:default.log
15672025-04-22T03:16:49.692Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
15682025-04-22T03:16:50.792Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:default.log
15692025-04-22T03:16:50.792Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:downstairs-47c499d8-e8b1-44ec-8988-927de21b5b7b.log (3007 bytes)
15702025-04-22T03:16:51.889Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:downstairs-47c499d8-e8b1-44ec-8988-927de21b5b7b.log
15712025-04-22T03:16:51.914Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:downstairs-734ae9f5-11f0-4920-abc3-2e0ac6103d0c.log (5974 bytes)
15722025-04-22T03:16:52.988Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:downstairs-734ae9f5-11f0-4920-abc3-2e0ac6103d0c.log
15732025-04-22T03:16:52.989Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:downstairs-cfaff0b5-d8b8-428d-bc85-c8044daddb69.log (6078 bytes)
15742025-04-22T03:16:54.102Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:downstairs-cfaff0b5-d8b8-428d-bc85-c8044daddb69.log
15752025-04-22T03:16:54.126Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:snapshot-cfaff0b5-d8b8-428d-bc85-c8044daddb69-215b9cb3-2af4-4667-a31f-61646d6a4191.log (5657 bytes)
15762025-04-22T03:16:55.219Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-crucible-downstairs:snapshot-cfaff0b5-d8b8-428d-bc85-c8044daddb69-215b9cb3-2af4-4667-a31f-61646d6a4191.log
15772025-04-22T03:16:55.219Zuploading: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-zone-network-setup:default.log (2083 bytes)
15782025-04-22T03:16:56.351Zuploaded: /pool/ext/ddbf788c-925c-40e0-9288-109e29e9e4a9/crypt/zone/oxz_crucible_d4a9a9ea-9881-4349-b52c-eb1157e57f59/root/var/svc/log/oxide-zone-network-setup:default.log
15792025-04-22T03:16:56.378Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/measurements.log (434936 bytes)
15802025-04-22T03:16:56.659Zupload warning: file "/pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/measurements.log" changed size mid upload: 434936 -> 532898
15812025-04-22T03:16:57.733Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/measurements.log
15822025-04-22T03:16:57.733Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/statistics.log (352755 bytes)
15832025-04-22T03:16:57.997Zupload warning: file "/pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/statistics.log" changed size mid upload: 352755 -> 435474
15842025-04-22T03:16:59.091Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/statistics.log
15852025-04-22T03:16:59.091Zuploading: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/tracking.log (47668 bytes)
15862025-04-22T03:16:59.239Zupload warning: file "/pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/tracking.log" changed size mid upload: 47668 -> 56776
15872025-04-22T03:17:00.314Zuploaded: /pool/ext/83f4d6e1-915c-4d7f-beea-b28c7a084ba3/crypt/zone/oxz_ntp_ca77233d-aaea-47f1-9896-c13aa0ab0ad6/root/var/log/chrony/tracking.log