01HMASP48NPXN46M1H3T6BCDF4: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-sled-agent:default.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.27",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HMASQR410Z3EMAQET1RN988A

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-01-17T05:00:13.590Zjob dependencies complete; ready to run (waiting for 50 m 59 s)
22024-01-17T05:04:48.841Zjob assigned to worker 01HMAWKSMRBH85Y3T3PEQKSZA1 (queued for 4 m 35 s)
32024-01-17T05:04:49.131Zhttp://172.20.0.5:9969/ipxe/sock... ok
42024-01-17T05:04:49.291Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52024-01-17T05:04:49.499Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... 21%Error: No response to keepalive - Terminating session
62024-01-17T05:04:49.798ZError: No response de-activating SOL payload
72024-01-17T05:04:50.029Z[SOL Session operational. Use ~? for help]
82024-01-17T05:05:09.911Z 89%Error: No response to keepalive - Terminating session
92024-01-17T05:05:11.264Zdownloading input: /input/ci-tools/work/end-to-end-tests/bootstrap-74528a6427e55c08.gz
102024-01-17T05:05:12.660Zdownloaded input: /input/ci-tools/work/end-to-end-tests/bootstrap-74528a6427e55c08.gz
112024-01-17T05:05:12.660Zdownloading input: /input/ci-tools/work/end-to-end-tests/bootstrap.gz
122024-01-17T05:05:13.816ZError: No response de-activating SOL payload
132024-01-17T05:05:15.119Z[SOL Session operational. Use ~? for help]
142024-01-17T05:05:33.060Zdownloaded input: /input/ci-tools/work/end-to-end-tests/bootstrap.gz
152024-01-17T05:05:33.060Zdownloading input: /input/ci-tools/work/end-to-end-tests/end_to_end_tests-346e854c34211b51.gz
162024-01-17T05:05:52.576Zdownloaded input: /input/ci-tools/work/end-to-end-tests/end_to_end_tests-346e854c34211b51.gz
172024-01-17T05:05:52.576Zdownloading input: /input/ci-tools/work/caboose-util.gz
182024-01-17T05:05:53.995Zdownloaded input: /input/ci-tools/work/caboose-util.gz
192024-01-17T05:05:53.995Zdownloading input: /input/ci-tools/work/tufaceous.gz
202024-01-17T05:05:57.456Zdownloaded input: /input/ci-tools/work/tufaceous.gz
212024-01-17T05:05:57.481Zdownloading input: /input/package/work/version.txt
222024-01-17T05:05:57.511Zdownloaded input: /input/package/work/version.txt
232024-01-17T05:05:57.511Zdownloading input: /input/package/work/package.tar.gz
242024-01-17T05:06:13.858Zdownloaded input: /input/package/work/package.tar.gz
252024-01-17T05:06:13.858Zdownloading input: /input/package/work/global-zone-packages.tar.gz
262024-01-17T05:06:40.629Zdownloaded input: /input/package/work/global-zone-packages.tar.gz
272024-01-17T05:06:40.629Zdownloading input: /input/package/work/trampoline-global-zone-packages.tar.gz
282024-01-17T05:06:50.120Zdownloaded input: /input/package/work/trampoline-global-zone-packages.tar.gz
292024-01-17T05:06:50.172Zdownloading input: /input/package/work/zones/clickhouse.tar.gz
302024-01-17T05:07:37.493Zdownloaded input: /input/package/work/zones/clickhouse.tar.gz
312024-01-17T05:07:37.494Zdownloading input: /input/package/work/zones/clickhouse_keeper.tar.gz
322024-01-17T05:08:03.620ZError: No response to keepalive - Terminating session
332024-01-17T05:08:13.667ZError: No response de-activating SOL payload
342024-01-17T05:08:15.034Z[SOL Session operational. Use ~? for help]
352024-01-17T05:08:26.509Zdownloaded input: /input/package/work/zones/clickhouse_keeper.tar.gz
362024-01-17T05:08:26.509Zdownloading input: /input/package/work/zones/cockroachdb.tar.gz
372024-01-17T05:08:57.801Zdownloaded input: /input/package/work/zones/cockroachdb.tar.gz
382024-01-17T05:08:57.801Zdownloading input: /input/package/work/zones/crucible-pantry.tar.gz
392024-01-17T05:09:04.796Zdownloaded input: /input/package/work/zones/crucible-pantry.tar.gz
402024-01-17T05:09:04.796Zdownloading input: /input/package/work/zones/crucible.tar.gz
412024-01-17T05:09:11.973Zdownloaded input: /input/package/work/zones/crucible.tar.gz
422024-01-17T05:09:12.413Zdownloading input: /input/package/work/zones/external-dns.tar.gz
432024-01-17T05:09:17.451Zdownloaded input: /input/package/work/zones/external-dns.tar.gz
442024-01-17T05:09:17.451Zdownloading input: /input/package/work/zones/internal-dns.tar.gz
452024-01-17T05:09:22.906Zdownloaded input: /input/package/work/zones/internal-dns.tar.gz
462024-01-17T05:09:22.907Zdownloading input: /input/package/work/zones/ntp.tar.gz
472024-01-17T05:09:22.952Zdownloaded input: /input/package/work/zones/ntp.tar.gz
482024-01-17T05:09:22.953Zdownloading input: /input/package/work/zones/omicron-gateway-asic.tar.gz
492024-01-17T05:09:27.846Zdownloaded input: /input/package/work/zones/omicron-gateway-asic.tar.gz
502024-01-17T05:09:27.878Zdownloading input: /input/package/work/zones/omicron-gateway-softnpu.tar.gz
512024-01-17T05:09:32.613Zdownloaded input: /input/package/work/zones/omicron-gateway-softnpu.tar.gz
522024-01-17T05:09:32.645Zdownloading input: /input/package/work/zones/omicron-nexus-single-sled.tar.gz
532024-01-17T05:09:52.788Zdownloaded input: /input/package/work/zones/omicron-nexus-single-sled.tar.gz
542024-01-17T05:09:52.789Zdownloading input: /input/package/work/zones/omicron-nexus.tar.gz
552024-01-17T05:10:07.752Zdownloaded input: /input/package/work/zones/omicron-nexus.tar.gz
562024-01-17T05:10:07.784Zdownloading input: /input/package/work/zones/overlay.tar.gz
572024-01-17T05:10:07.834Zdownloaded input: /input/package/work/zones/overlay.tar.gz
582024-01-17T05:10:07.868Zdownloading input: /input/package/work/zones/oximeter-collector.tar.gz
592024-01-17T05:10:17.033Zdownloaded input: /input/package/work/zones/oximeter-collector.tar.gz
602024-01-17T05:10:17.067Zdownloading input: /input/package/work/zones/propolis-server.tar.gz
612024-01-17T05:10:31.252Zdownloaded input: /input/package/work/zones/propolis-server.tar.gz
622024-01-17T05:10:31.291Zdownloading input: /input/package/work/zones/switch-asic.tar.gz
632024-01-17T05:11:19.685ZError: No response to keepalive - Terminating session
642024-01-17T05:11:25.840Zdownloaded input: /input/package/work/zones/switch-asic.tar.gz
652024-01-17T05:11:25.871Zdownloading input: /input/package/work/zones/switch-softnpu.tar.gz
662024-01-17T05:11:29.729ZError: No response de-activating SOL payload
672024-01-17T05:11:30.874Z[SOL Session operational. Use ~? for help]
682024-01-17T05:12:10.329Zdownloaded input: /input/package/work/zones/switch-softnpu.tar.gz
 
692024-01-17T05:12:10.329Zstarting task 0: "setup"
702024-01-17T05:12:10.361Z++ uname -s
712024-01-17T05:12:10.361Z+ kern=SunOS
722024-01-17T05:12:10.362Z+ case "$kern" in
732024-01-17T05:12:10.362Z+ groupadd -g 12345 build
742024-01-17T05:12:10.362Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
752024-01-17T05:12:12.379Z+ zfs create -o mountpoint=/work rpool/work
762024-01-17T05:12:12.416Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
772024-01-17T05:12:12.416Z+ home_fs=zfs
782024-01-17T05:12:12.416Z+ [[ zfs == autofs ]]
792024-01-17T05:12:12.416Z+ mkdir -p /home/build
802024-01-17T05:12:12.416Z+ chown build:build /home/build /work
812024-01-17T05:12:14.390Z+ chmod 0700 /home/build /work
822024-01-17T05:12:14.478Zprocess exited: duration 4062 ms, exit code 0
 
832024-01-17T05:12:14.549Zstarting task 1: "authentication"
842024-01-17T05:12:14.595Zprocess exited: duration 44 ms, exit code 0
 
852024-01-17T05:12:14.643Zstarting task 2: "build"
862024-01-17T05:12:14.667Z+ trap _exit_trap EXIT
872024-01-17T05:12:14.667Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
882024-01-17T05:12:14.847Zsecpolicy_fs_linkdir: 0x55 = 0x55
892024-01-17T05:12:14.847Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
902024-01-17T05:12:14.848Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
912024-01-17T05:12:14.848Zsecpolicy_fs_linkdir+3: 0 = 0xe5
922024-01-17T05:12:14.848Zsecpolicy_fs_linkdir+4: 0 = 0xb8
932024-01-17T05:12:14.848Zsecpolicy_fs_linkdir+5: 0 = 0x1
942024-01-17T05:12:14.848Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
952024-01-17T05:12:14.849Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
962024-01-17T05:12:14.849Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
972024-01-17T05:12:14.885Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
982024-01-17T05:12:14.885Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
992024-01-17T05:12:14.885Z+ [[ -d /opt/oxide ]]
1002024-01-17T05:12:14.886Z+ pfexec mv /opt/oxide /opt/oxide-underneath
1012024-01-17T05:12:14.886Z+ pfexec mkdir /opt/oxide
1022024-01-17T05:12:14.886Z+ pfexec mount -F tmpfs -O swap /opt/oxide
1032024-01-17T05:12:14.886Z+ [[ -d /opt/oxide-underneath ]]
1042024-01-17T05:12:14.886Z+ cd /opt/oxide-underneath
1052024-01-17T05:12:14.886Z+ pfexec tar ceEp@/f - .
1062024-01-17T05:12:14.886Z+ cd /opt/oxide
1072024-01-17T05:12:14.886Z+ pfexec tar xveEp@/f -
1082024-01-17T05:12:14.974Zx ., 0 bytes, 0 tape blocks
1092024-01-17T05:12:14.974Zx ./opte, 0 bytes, 0 tape blocks
1102024-01-17T05:12:14.975Zx ./opte/bin, 0 bytes, 0 tape blocks
1112024-01-17T05:12:14.975Zx ./opte/bin/opteadm, 52567720 bytes, 102672 tape blocks
1122024-01-17T05:12:14.999Z+ pfexec rm -rf /opt/oxide-underneath
1132024-01-17T05:12:14.999Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
1142024-01-17T05:12:15.024Z+ pfexec mkdir /opt/oxide/work
1152024-01-17T05:12:15.024Z+ pfexec chown build:build /opt/oxide/work
1162024-01-17T05:12:15.024Z+ cd /opt/oxide/work
1172024-01-17T05:12:15.024Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
1182024-01-17T05:12:16.559ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
1192024-01-17T05:12:16.559Zx out/mg-ddm-gz.tar, 62723072 bytes, 122506 tape blocks
1202024-01-17T05:12:16.559Zx out/omicron-sled-agent.tar, 114633216 bytes, 223893 tape blocks
1212024-01-17T05:12:16.559Zx out/target/test, 75 bytes, 1 tape blocks
1222024-01-17T05:12:16.559Zx out/npuzone/npuzone, 37184104 bytes, 72626 tape blocks
1232024-01-17T05:12:16.559Zx package-manifest.toml, 22919 bytes, 45 tape blocks
1242024-01-17T05:12:16.559Zx smf/sled-agent/non-gimlet/config.toml, 3448 bytes, 7 tape blocks
1252024-01-17T05:12:16.561Zx target/release/omicron-package, 43081600 bytes, 84144 tape blocks
1262024-01-17T05:12:16.561Zx tools/create_virtual_hardware.sh, 3176 bytes, 7 tape blocks
1272024-01-17T05:12:16.561Zx tools/virtual_hardware.sh, 3408 bytes, 7 tape blocks
1282024-01-17T05:12:16.561Zx tools/scrimlet/create-softnpu-zone.sh, 619 bytes, 2 tape blocks
1292024-01-17T05:12:16.561Zx tools/scrimlet/destroy-softnpu-zone.sh, 195 bytes, 1 tape blocks
1302024-01-17T05:12:16.561Zx tools/scrimlet/softnpu-init.sh, 2149 bytes, 5 tape blocks
1312024-01-17T05:12:16.561Zx tools/scrimlet/softnpu-zone.txt, 330 bytes, 1 tape blocks
1322024-01-17T05:12:16.561Zx tools/scrimlet/softnpu.toml, 171 bytes, 1 tape blocks
1332024-01-17T05:12:16.586Z
1342024-01-17T05:12:16.586Zreal 1.547138209
1352024-01-17T05:12:16.586Zuser 1.396191968
1362024-01-17T05:12:16.586Zsys 0.267279290
1372024-01-17T05:12:16.586Ztrap 0.000070591
1382024-01-17T05:12:16.586Ztflt 0.000000000
1392024-01-17T05:12:16.586Zdflt 0.000000000
1402024-01-17T05:12:16.586Zkflt 0.000000000
1412024-01-17T05:12:16.586Zlock 0.000000000
1422024-01-17T05:12:16.586Zslp 1.394746453
1432024-01-17T05:12:16.586Zlat 0.034134445
1442024-01-17T05:12:16.586Zstop 0.000120022
1452024-01-17T05:12:16.586Z+ cp /input/package/work/zones/clickhouse_keeper.tar.gz /input/package/work/zones/clickhouse.tar.gz /input/package/work/zones/cockroachdb.tar.gz /input/package/work/zones/crucible-pantry.tar.gz /input/package/work/zones/crucible.tar.gz /input/package/work/zones/external-dns.tar.gz /input/package/work/zones/internal-dns.tar.gz /input/package/work/zones/ntp.tar.gz /input/package/work/zones/omicron-gateway-asic.tar.gz /input/package/work/zones/omicron-gateway-softnpu.tar.gz /input/package/work/zones/omicron-nexus-single-sled.tar.gz /input/package/work/zones/omicron-nexus.tar.gz /input/package/work/zones/overlay.tar.gz /input/package/work/zones/oximeter-collector.tar.gz /input/package/work/zones/propolis-server.tar.gz /input/package/work/zones/switch-asic.tar.gz /input/package/work/zones/switch-softnpu.tar.gz out/
1462024-01-17T05:12:17.698Z+ mv out/omicron-nexus-single-sled.tar.gz out/omicron-nexus.tar.gz
1472024-01-17T05:12:17.722Z+ mkdir tests
1482024-01-17T05:12:17.722Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1492024-01-17T05:12:17.722Z+ ptime -m gunzip
1502024-01-17T05:12:17.722Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap-74528a6427e55c08
1512024-01-17T05:12:17.858Z
1522024-01-17T05:12:17.883Zreal 0.140784045
1532024-01-17T05:12:17.883Zuser 0.127565605
1542024-01-17T05:12:17.884Zsys 0.012974915
1552024-01-17T05:12:17.884Ztrap 0.000012259
1562024-01-17T05:12:17.884Ztflt 0.000000000
1572024-01-17T05:12:17.884Zdflt 0.000000000
1582024-01-17T05:12:17.884Zkflt 0.000000000
1592024-01-17T05:12:17.884Zlock 0.000000000
1602024-01-17T05:12:17.884Zslp 0.000020020
1612024-01-17T05:12:17.884Zlat 0.000023550
1622024-01-17T05:12:17.884Zstop 0.000129602
1632024-01-17T05:12:17.884Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap-74528a6427e55c08
1642024-01-17T05:12:17.884Z+ chmod a+x tests/bootstrap-74528a6427e55c08
1652024-01-17T05:12:17.884Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1662024-01-17T05:12:17.884Z+ ptime -m gunzip
1672024-01-17T05:12:17.884Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap
1682024-01-17T05:12:20.136Z
1692024-01-17T05:12:20.160Zreal 2.265903753
1702024-01-17T05:12:20.160Zuser 2.130149114
1712024-01-17T05:12:20.160Zsys 0.135365716
1722024-01-17T05:12:20.162Ztrap 0.000156411
1732024-01-17T05:12:20.162Ztflt 0.000000000
1742024-01-17T05:12:20.162Zdflt 0.000000000
1752024-01-17T05:12:20.162Zkflt 0.000000000
1762024-01-17T05:12:20.162Zlock 0.000000000
1772024-01-17T05:12:20.162Zslp 0.000021270
1782024-01-17T05:12:20.162Zlat 0.000023689
1792024-01-17T05:12:20.162Zstop 0.000121572
1802024-01-17T05:12:20.162Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap
1812024-01-17T05:12:20.162Z+ chmod a+x tests/bootstrap
1822024-01-17T05:12:20.162Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1832024-01-17T05:12:20.162Z+ ptime -m gunzip
1842024-01-17T05:12:20.163Z++ basename /input/ci-tools/work/end-to-end-tests/end_to_end_tests-346e854c34211b51
1852024-01-17T05:12:24.951Z
1862024-01-17T05:12:25.005Zreal 2.350568264
1872024-01-17T05:12:25.005Zuser 2.190512421
1882024-01-17T05:12:25.005Zsys 0.159633373
1892024-01-17T05:12:25.005Ztrap 0.000168134
1902024-01-17T05:12:25.005Ztflt 0.000000000
1912024-01-17T05:12:25.005Zdflt 0.000000000
1922024-01-17T05:12:25.006Zkflt 0.000000000
1932024-01-17T05:12:25.006Zlock 0.000000000
1942024-01-17T05:12:25.006Zslp 0.000020270
1952024-01-17T05:12:25.006Zlat 0.000020710
1962024-01-17T05:12:25.006Zstop 0.000120012
1972024-01-17T05:12:25.006Z++ basename /input/ci-tools/work/end-to-end-tests/end_to_end_tests-346e854c34211b51
1982024-01-17T05:12:25.006Z+ chmod a+x tests/end_to_end_tests-346e854c34211b51
1992024-01-17T05:12:25.006Z+ bmat address ls
2002024-01-17T05:12:25.006ZNAME CIDR FIRST LAST COUNT
2012024-01-17T05:12:25.006Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
2022024-01-17T05:12:25.006Z++ bmat address ls -f extra -Ho first,last,count
2032024-01-17T05:12:25.006Z+ set -- 10.151.2.164 10.151.2.227 64
2042024-01-17T05:12:25.007Z+ EXTRA_IP_START=10.151.2.164
2052024-01-17T05:12:25.007Z+ EXTRA_IP_END=10.151.2.227
2062024-01-17T05:12:25.007Z+ EXTRA_IP_COUNT=64
2072024-01-17T05:12:25.007Z+ (( EXTRA_IP_COUNT >= 32 ))
2082024-01-17T05:12:25.007Z+ EXTRA_IP_BASE=10.151.2
2092024-01-17T05:12:25.008Z+ EXTRA_IP_FOCTET=164
2102024-01-17T05:12:25.008Z+ EXTRA_IP_LOCTET=227
2112024-01-17T05:12:25.008Z+ SERVICE_IP_POOL_START=10.151.2.164
2122024-01-17T05:12:25.008Z+ SERVICE_IP_POOL_END=10.151.2.173
2132024-01-17T05:12:25.008Z+ DNS_IP1=10.151.2.164
2142024-01-17T05:12:25.008Z+ DNS_IP2=10.151.2.165
2152024-01-17T05:12:25.008Z+ UPLINK_IP=10.151.2.174
2162024-01-17T05:12:25.008Z+ IPPOOL_START=10.151.2.175
2172024-01-17T05:12:25.008Z+ IPPOOL_END=10.151.2.227
2182024-01-17T05:12:25.009Z++ ipadm show-addr -po type,addr
2192024-01-17T05:12:25.009Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
2202024-01-17T05:12:25.009Z+ GATEWAY_IP=10.151.2.100
2212024-01-17T05:12:25.009Z+ [[ -n 10.151.2.100 ]]
2222024-01-17T05:12:25.009Z+ ping -s 10.151.2.100 56 1
2232024-01-17T05:12:25.009ZPING 10.151.2.100: 56 data bytes
2242024-01-17T05:12:25.009Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.010 ms
2252024-01-17T05:12:25.009Z
2262024-01-17T05:12:25.009Z----10.151.2.100 PING Statistics----
2272024-01-17T05:12:25.009Z1 packets transmitted, 1 packets received, 0% packet loss
2282024-01-17T05:12:25.009Zround-trip (ms) min/avg/max/stddev = 0.010/0.010/0.010/-nan
2292024-01-17T05:12:25.009Z++ arp -an
2302024-01-17T05:12:25.010Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
2312024-01-17T05:12:25.010Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
2322024-01-17T05:12:25.010Z+ routeadm -e ipv4-forwarding -u
2332024-01-17T05:12:25.010Z+ PXA_START=10.151.2.164
2342024-01-17T05:12:25.010Z+ PXA_END=10.151.2.227
2352024-01-17T05:12:25.010Z+ export GATEWAY_IP GATEWAY_MAC PXA_START PXA_END
2362024-01-17T05:12:25.010Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
2372024-01-17T05:12:25.011Z+ ZPOOL_VDEV_DIR=/scratch
2382024-01-17T05:12:25.011Z+ ptime -m pfexec ./tools/create_virtual_hardware.sh
2392024-01-17T05:12:25.011Z+++ dirname ./tools/create_virtual_hardware.sh
2402024-01-17T05:12:25.012Z++ cd ./tools
2412024-01-17T05:12:25.012Z++ pwd
2422024-01-17T05:12:25.012Z+ SOURCE_DIR=/opt/oxide/work/tools
2432024-01-17T05:12:25.012Z+ OMICRON_TOP=/opt/oxide/work/tools/..
2442024-01-17T05:12:25.012Z+ SOFTNPU_MODE=zone
2452024-01-17T05:12:25.012Z+ . /opt/oxide/work/tools/virtual_hardware.sh
2462024-01-17T05:12:25.012Z++ MARKER=/etc/opt/oxide/NO_INSTALL
2472024-01-17T05:12:25.012Z++ [[ -f /etc/opt/oxide/NO_INSTALL ]]
2482024-01-17T05:12:25.012Z++ dladm show-phys -p -o LINK
2492024-01-17T05:12:25.013Z++ head -1
2502024-01-17T05:12:25.013Z+ PHYSICAL_LINK=igb0
2512024-01-17T05:12:25.013Z+ echo 'Using igb0 as physical link'
2522024-01-17T05:12:25.013ZUsing igb0 as physical link
2532024-01-17T05:12:25.013Z+ ensure_run_as_root
2542024-01-17T05:12:25.013Z++ id -u
2552024-01-17T05:12:25.013Z+ [[ 0 -ne 0 ]]
2562024-01-17T05:12:25.013Z+ ensure_zpools
2572024-01-17T05:12:25.013Z+ ZPOOL_TYPES=('oxp_' 'oxi_')
2582024-01-17T05:12:25.013Z+ for ZPOOL_TYPE in "${ZPOOL_TYPES[@]}"
2592024-01-17T05:12:25.015Z+ readarray -t ZPOOLS
2602024-01-17T05:12:25.015Z++ grep '"oxp_' /opt/oxide/work/tools/../smf/sled-agent/non-gimlet/config.toml
2612024-01-17T05:12:25.015Z++ sed 's/[ ",]//g'
2622024-01-17T05:12:25.015Z+ for ZPOOL in "${ZPOOLS[@]}"
2632024-01-17T05:12:25.015Z+ echo 'Zpool: [oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b]'
2642024-01-17T05:12:25.015ZZpool: [oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b]
2652024-01-17T05:12:25.017Z+ VDEV_PATH=/scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
2662024-01-17T05:12:25.018Z+ [[ -f /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev ]]
2672024-01-17T05:12:25.018Z+ dd if=/dev/zero of=/scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev bs=1 count=0 seek=20G
2682024-01-17T05:12:25.018Z0+0 records in
2692024-01-17T05:12:25.018Z0+0 records out
2702024-01-17T05:12:25.018Z0 bytes (0 iB) transferred in 0.000008 secs (0 iB/sec)
2712024-01-17T05:12:25.018Z+ success 'ZFS vdev /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists'
2722024-01-17T05:12:25.018Z+ set +x
2732024-01-17T05:12:25.018ZZFS vdev /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists
2742024-01-17T05:12:25.018Z++ zpool list -o name
2752024-01-17T05:12:25.018Z++ grep oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
2762024-01-17T05:12:25.018Z+ [[ -z '' ]]
2772024-01-17T05:12:25.018Z+ zpool create -f oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
2782024-01-17T05:12:25.019Z+ success 'ZFS zpool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b exists'
2792024-01-17T05:12:25.019Z+ set +x
2802024-01-17T05:12:25.019ZZFS zpool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b exists
2812024-01-17T05:12:25.019ZZpool: [oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03]
2822024-01-17T05:12:25.019Z+ for ZPOOL in "${ZPOOLS[@]}"
2832024-01-17T05:12:25.019Z+ echo 'Zpool: [oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03]'
2842024-01-17T05:12:25.019Z+ VDEV_PATH=/scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
2852024-01-17T05:12:25.019Z+ [[ -f /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
2862024-01-17T05:12:25.075Z+ dd if=/dev/zero of=/scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
2872024-01-17T05:12:25.075Z0+0 records in
2882024-01-17T05:12:25.075Z0+0 records out
2892024-01-17T05:12:25.075Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
2902024-01-17T05:12:25.076Z+ success 'ZFS vdev /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
2912024-01-17T05:12:25.077Z+ set +x
2922024-01-17T05:12:25.077ZZFS vdev /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
2932024-01-17T05:12:25.077Z++ zpool list -o name
2942024-01-17T05:12:25.077Z++ grep oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
2952024-01-17T05:12:25.077Z+ [[ -z '' ]]
2962024-01-17T05:12:25.077Z+ zpool create -f oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
2972024-01-17T05:12:25.077Z+ success 'ZFS zpool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
2982024-01-17T05:12:25.077Z+ set +x
2992024-01-17T05:12:25.077ZZFS zpool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
3002024-01-17T05:12:25.077Z+ for ZPOOL in "${ZPOOLS[@]}"
3012024-01-17T05:12:25.078Z+ echo 'Zpool: [oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03]'
3022024-01-17T05:12:25.078ZZpool: [oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03]
3032024-01-17T05:12:25.078Z+ VDEV_PATH=/scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3042024-01-17T05:12:25.078Z+ [[ -f /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3052024-01-17T05:12:25.078Z+ dd if=/dev/zero of=/scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3062024-01-17T05:12:25.078Z0+0 records in
3072024-01-17T05:12:25.078Z0+0 records out
3082024-01-17T05:12:25.078Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
3092024-01-17T05:12:25.078Z+ success 'ZFS vdev /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3102024-01-17T05:12:25.078Z+ set +x
3112024-01-17T05:12:25.078ZZFS vdev /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
3122024-01-17T05:12:25.078Z++ zpool list -o name
3132024-01-17T05:12:25.079Z++ grep oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
3142024-01-17T05:12:25.079Z+ [[ -z '' ]]
3152024-01-17T05:12:25.081Z+ zpool create -f oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3162024-01-17T05:12:25.081Z+ success 'ZFS zpool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3172024-01-17T05:12:25.081Z+ set +x
3182024-01-17T05:12:25.081ZZFS zpool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
3192024-01-17T05:12:25.081Z+ for ZPOOL in "${ZPOOLS[@]}"
3202024-01-17T05:12:25.081Z+ echo 'Zpool: [oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03]'
3212024-01-17T05:12:25.081ZZpool: [oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03]
3222024-01-17T05:12:25.081Z+ VDEV_PATH=/scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3232024-01-17T05:12:25.081Z+ [[ -f /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3242024-01-17T05:12:25.081Z+ dd if=/dev/zero of=/scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3252024-01-17T05:12:25.082Z0+0 records in
3262024-01-17T05:12:25.082Z0+0 records out
3272024-01-17T05:12:25.082Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
3282024-01-17T05:12:25.082Z+ success 'ZFS vdev /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3292024-01-17T05:12:25.082Z+ set +x
3302024-01-17T05:12:25.082ZZFS vdev /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
3312024-01-17T05:12:25.082Z++ zpool list -o name
3322024-01-17T05:12:25.082Z++ grep oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03
3332024-01-17T05:12:25.082Z+ [[ -z '' ]]
3342024-01-17T05:12:25.082Z+ zpool create -f oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3352024-01-17T05:12:25.082Z+ success 'ZFS zpool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3362024-01-17T05:12:25.083Z+ set +x
3372024-01-17T05:12:25.083ZZFS zpool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
3382024-01-17T05:12:25.083Z+ for ZPOOL in "${ZPOOLS[@]}"
3392024-01-17T05:12:25.083Z+ echo 'Zpool: [oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03]'
3402024-01-17T05:12:25.085ZZpool: [oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03]
3412024-01-17T05:12:25.085Z+ VDEV_PATH=/scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3422024-01-17T05:12:25.085Z+ [[ -f /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3432024-01-17T05:12:25.085Z+ dd if=/dev/zero of=/scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3442024-01-17T05:12:25.085Z0+0 records in
3452024-01-17T05:12:25.085Z0+0 records out
3462024-01-17T05:12:25.085Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
3472024-01-17T05:12:25.085Z+ success 'ZFS vdev /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3482024-01-17T05:12:25.085Z+ set +x
3492024-01-17T05:12:25.085ZZFS vdev /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
3502024-01-17T05:12:25.085Z++ zpool list -o name
3512024-01-17T05:12:25.085Z++ grep oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03
3522024-01-17T05:12:25.085Z+ [[ -z '' ]]
3532024-01-17T05:12:25.085Z+ zpool create -f oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3542024-01-17T05:12:25.085Z+ success 'ZFS zpool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3552024-01-17T05:12:25.086Z+ set +x
3562024-01-17T05:12:25.086ZZFS zpool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
3572024-01-17T05:12:25.086Z+ for ZPOOL in "${ZPOOLS[@]}"
3582024-01-17T05:12:25.086Z+ echo 'Zpool: [oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930]'
3592024-01-17T05:12:25.086ZZpool: [oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930]
3602024-01-17T05:12:25.086Z+ VDEV_PATH=/scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev
3612024-01-17T05:12:25.086Z+ [[ -f /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev ]]
3622024-01-17T05:12:25.086Z+ dd if=/dev/zero of=/scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev bs=1 count=0 seek=20G
3632024-01-17T05:12:25.086Z0+0 records in
3642024-01-17T05:12:25.087Z0+0 records out
3652024-01-17T05:12:25.087Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
3662024-01-17T05:12:25.087Z+ success 'ZFS vdev /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev exists'
3672024-01-17T05:12:25.087Z+ set +x
3682024-01-17T05:12:25.087ZZFS vdev /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev exists
3692024-01-17T05:12:25.087Z++ zpool list -o name
3702024-01-17T05:12:25.087Z++ grep oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930
3712024-01-17T05:12:25.087Z+ [[ -z '' ]]
3722024-01-17T05:12:25.087Z+ zpool create -f oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev
3732024-01-17T05:12:25.087Z+ success 'ZFS zpool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 exists'
3742024-01-17T05:12:25.087Z+ set +x
3752024-01-17T05:12:25.088ZZFS zpool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 exists
3762024-01-17T05:12:25.088Z+ for ZPOOL in "${ZPOOLS[@]}"
3772024-01-17T05:12:25.088ZZpool: [oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60]
3782024-01-17T05:12:25.088Z+ echo 'Zpool: [oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60]'
3792024-01-17T05:12:25.088Z+ VDEV_PATH=/scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev
3802024-01-17T05:12:25.088Z+ [[ -f /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev ]]
3812024-01-17T05:12:25.088Z+ dd if=/dev/zero of=/scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev bs=1 count=0 seek=20G
3822024-01-17T05:12:25.088Z0+0 records in
3832024-01-17T05:12:25.088Z0+0 records out
3842024-01-17T05:12:25.088Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
3852024-01-17T05:12:25.088Z+ success 'ZFS vdev /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev exists'
3862024-01-17T05:12:25.129Z+ set +x
3872024-01-17T05:12:25.129ZZFS vdev /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev exists
3882024-01-17T05:12:25.129Z++ zpool list -o name
3892024-01-17T05:12:25.129Z++ grep oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60
3902024-01-17T05:12:25.129Z+ [[ -z '' ]]
3912024-01-17T05:12:25.129Z+ zpool create -f oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev
3922024-01-17T05:12:25.129Z+ success 'ZFS zpool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 exists'
3932024-01-17T05:12:25.129Z+ set +x
3942024-01-17T05:12:25.129ZZFS zpool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 exists
3952024-01-17T05:12:25.129Z+ for ZPOOL in "${ZPOOLS[@]}"
3962024-01-17T05:12:25.129Z+ echo 'Zpool: [oxp_31bd71cd-4736-4a12-a387-9b74b050396f]'
3972024-01-17T05:12:25.130ZZpool: [oxp_31bd71cd-4736-4a12-a387-9b74b050396f]
3982024-01-17T05:12:25.130Z+ VDEV_PATH=/scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev
3992024-01-17T05:12:25.130Z+ [[ -f /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev ]]
4002024-01-17T05:12:25.130Z+ dd if=/dev/zero of=/scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev bs=1 count=0 seek=20G
4012024-01-17T05:12:25.130Z0+0 records in
4022024-01-17T05:12:25.130Z0+0 records out
4032024-01-17T05:12:25.130Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
4042024-01-17T05:12:25.130Z+ success 'ZFS vdev /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev exists'
4052024-01-17T05:12:25.130Z+ set +x
4062024-01-17T05:12:25.131ZZFS vdev /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev exists
4072024-01-17T05:12:25.131Z++ zpool list -o name
4082024-01-17T05:12:25.131Z++ grep oxp_31bd71cd-4736-4a12-a387-9b74b050396f
4092024-01-17T05:12:25.131Z+ [[ -z '' ]]
4102024-01-17T05:12:25.131Z+ zpool create -f oxp_31bd71cd-4736-4a12-a387-9b74b050396f /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev
4112024-01-17T05:12:25.131Z+ success 'ZFS zpool oxp_31bd71cd-4736-4a12-a387-9b74b050396f exists'
4122024-01-17T05:12:25.131Z+ set +x
4132024-01-17T05:12:25.131ZZFS zpool oxp_31bd71cd-4736-4a12-a387-9b74b050396f exists
4142024-01-17T05:12:25.132Z+ for ZPOOL in "${ZPOOLS[@]}"
4152024-01-17T05:12:25.132Z+ echo 'Zpool: [oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7]'
4162024-01-17T05:12:25.132ZZpool: [oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7]
4172024-01-17T05:12:25.132Z+ VDEV_PATH=/scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev
4182024-01-17T05:12:25.132Z+ [[ -f /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev ]]
4192024-01-17T05:12:25.132Z+ dd if=/dev/zero of=/scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev bs=1 count=0 seek=20G
4202024-01-17T05:12:25.132Z0+0 records in
4212024-01-17T05:12:25.132Z0+0 records out
4222024-01-17T05:12:25.132Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4232024-01-17T05:12:25.132Z+ success 'ZFS vdev /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev exists'
4242024-01-17T05:12:25.133Z+ set +x
4252024-01-17T05:12:25.133ZZFS vdev /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev exists
4262024-01-17T05:12:25.133Z++ zpool list -o name
4272024-01-17T05:12:25.133Z++ grep oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7
4282024-01-17T05:12:25.133Z+ [[ -z '' ]]
4292024-01-17T05:12:25.133Z+ zpool create -f oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev
4302024-01-17T05:12:25.133Z+ success 'ZFS zpool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 exists'
4312024-01-17T05:12:25.133Z+ set +x
4322024-01-17T05:12:25.133ZZFS zpool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 exists
4332024-01-17T05:12:25.133Z+ for ZPOOL_TYPE in "${ZPOOL_TYPES[@]}"
4342024-01-17T05:12:25.133Z+ readarray -t ZPOOLS
4352024-01-17T05:12:25.133Z++ grep '"oxi_' /opt/oxide/work/tools/../smf/sled-agent/non-gimlet/config.toml
4362024-01-17T05:12:25.134Z++ sed 's/[ ",]//g'
4372024-01-17T05:12:25.134Z+ for ZPOOL in "${ZPOOLS[@]}"
4382024-01-17T05:12:25.134Z+ echo 'Zpool: [oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b]'
4392024-01-17T05:12:25.135ZZpool: [oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b]
4402024-01-17T05:12:25.135Z+ VDEV_PATH=/scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4412024-01-17T05:12:25.135Z+ [[ -f /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev ]]
4422024-01-17T05:12:25.135Z+ dd if=/dev/zero of=/scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev bs=1 count=0 seek=20G
4432024-01-17T05:12:25.136Z0+0 records in
4442024-01-17T05:12:25.136Z0+0 records out
4452024-01-17T05:12:25.137Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4462024-01-17T05:12:25.137Z+ success 'ZFS vdev /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists'
4472024-01-17T05:12:25.137Z+ set +x
4482024-01-17T05:12:25.137ZZFS vdev /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists
4492024-01-17T05:12:25.137Z++ zpool list -o name
4502024-01-17T05:12:25.138Z++ grep oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b
4512024-01-17T05:12:25.138Z+ [[ -z '' ]]
4522024-01-17T05:12:25.138Z+ zpool create -f oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4532024-01-17T05:12:25.138Z+ success 'ZFS zpool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b exists'
4542024-01-17T05:12:25.138Z+ set +x
4552024-01-17T05:12:25.138ZZFS zpool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b exists
4562024-01-17T05:12:25.138Z+ for ZPOOL in "${ZPOOLS[@]}"
4572024-01-17T05:12:25.138Z+ echo 'Zpool: [oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b]'
4582024-01-17T05:12:25.138ZZpool: [oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b]
4592024-01-17T05:12:25.138Z+ VDEV_PATH=/scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4602024-01-17T05:12:25.138Z+ [[ -f /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev ]]
4612024-01-17T05:12:25.138Z+ dd if=/dev/zero of=/scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev bs=1 count=0 seek=20G
4622024-01-17T05:12:25.138Z0+0 records in
4632024-01-17T05:12:25.140Z0+0 records out
4642024-01-17T05:12:25.140Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4652024-01-17T05:12:25.140Z+ success 'ZFS vdev /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists'
4662024-01-17T05:12:25.140Z+ set +x
4672024-01-17T05:12:25.140ZZFS vdev /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists
4682024-01-17T05:12:25.140Z++ zpool list -o name
4692024-01-17T05:12:25.140Z++ grep oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b
4702024-01-17T05:12:25.140Z+ [[ -z '' ]]
4712024-01-17T05:12:25.140Z+ zpool create -f oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4722024-01-17T05:12:25.140Z+ success 'ZFS zpool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b exists'
4732024-01-17T05:12:25.140Z+ set +x
4742024-01-17T05:12:25.141ZZFS zpool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b exists
4752024-01-17T05:12:25.141Z+ [[ zone == \z\o\n\e ]]
4762024-01-17T05:12:25.141Z+ ensure_simulated_links igb0
4772024-01-17T05:12:25.141Z+ local PHYSICAL_LINK=igb0
4782024-01-17T05:12:25.141Z+ INDICES=("0" "1")
4792024-01-17T05:12:25.141Z+ for I in "${INDICES[@]}"
4802024-01-17T05:12:25.141Z++ get_simnet_name_if_exists net0
4812024-01-17T05:12:25.141Z++ dladm show-simnet -p -o LINK net0
4822024-01-17T05:12:25.141Z++ echo ''
4832024-01-17T05:12:25.141Z+ [[ -z '' ]]
4842024-01-17T05:12:25.141Z+ dladm create-simnet -t net0
4852024-01-17T05:12:25.142Z+ dladm create-simnet -t sc0_0
4862024-01-17T05:12:25.169Z+ dladm modify-simnet -t -p net0 sc0_0
4872024-01-17T05:12:25.169Z+ dladm set-linkprop -p mtu=9000 sc0_0
4882024-01-17T05:12:25.169Z+ success 'Simnet net0/sc0_0 exists'
4892024-01-17T05:12:25.169Z+ set +x
4902024-01-17T05:12:25.170Z+ for I in "${INDICES[@]}"
4912024-01-17T05:12:25.171ZSimnet net0/sc0_0 exists
4922024-01-17T05:12:25.171Z++ get_simnet_name_if_exists net1
4932024-01-17T05:12:25.171Z++ dladm show-simnet -p -o LINK net1
4942024-01-17T05:12:25.171Z++ echo ''
4952024-01-17T05:12:25.171Z+ [[ -z '' ]]
4962024-01-17T05:12:25.171Z+ dladm create-simnet -t net1
4972024-01-17T05:12:25.171Z+ dladm create-simnet -t sc1_0
4982024-01-17T05:12:25.171Z+ dladm modify-simnet -t -p net1 sc1_0
4992024-01-17T05:12:25.171Z+ dladm set-linkprop -p mtu=9000 sc1_0
5002024-01-17T05:12:25.171Z+ success 'Simnet net1/sc1_0 exists'
5012024-01-17T05:12:25.171Z+ set +x
5022024-01-17T05:12:25.171ZSimnet net1/sc1_0 exists
5032024-01-17T05:12:25.171Z++ get_vnic_name_if_exists sc0_1
5042024-01-17T05:12:25.172Z++ dladm show-vnic -p -o LINK sc0_1
5052024-01-17T05:12:25.172Z++ echo ''
5062024-01-17T05:12:25.172Z+ [[ -z '' ]]
5072024-01-17T05:12:25.172Z+ dladm create-vnic -t sc0_1 -l igb0 -m a8:e1:de:01:70:1d
5082024-01-17T05:12:25.172Z+ [[ -v PROMISC_FILT_OFF ]]
5092024-01-17T05:12:25.172Z+ success 'Vnic sc0_1 exists'
5102024-01-17T05:12:25.172Z+ set +x
5112024-01-17T05:12:25.172ZVnic sc0_1 exists
5122024-01-17T05:12:25.172Z+ warn_if_no_proxy_arp
5132024-01-17T05:12:25.172Z+ [[ -v PXA_START ]]
5142024-01-17T05:12:25.172Z+ [[ -v PXA_END ]]
5152024-01-17T05:12:25.172Z+ ensure_softnpu_zone
5162024-01-17T05:12:25.172Z+ zoneadm list
5172024-01-17T05:12:25.172Z+ grep -q sidecar_softnpu
5182024-01-17T05:12:25.172Z+ [[ -f out/npuzone/npuzone ]]
5192024-01-17T05:12:25.172Z+ out/npuzone/npuzone create sidecar --omicron-zone --ports sc0_0,tfportrear0_0 --ports sc0_1,tfportqsfp0_0 --sidecar-lite-commit e3ea4b495ba0a71801ded0776ae4bbd31df57e26 --softnpu-commit dbab082dfa89da5db5ca2325c257089d2f130092
5202024-01-17T05:12:25.172Zremote url: https://buildomat.eng.oxide.computer/public/file/oxidecomputer/softnpu/image/dbab082dfa89da5db5ca2325c257089d2f130092/softnpu.sha256.txt
5212024-01-17T05:12:31.075Zdownloading softnpu ...done
5222024-01-17T05:12:31.099Zremote url: https://buildomat.eng.oxide.computer/public/file/oxidecomputer/sidecar-lite/release/e3ea4b495ba0a71801ded0776ae4bbd31df57e26/libsidecar_lite.so.sha256.txt
5232024-01-17T05:12:36.247Zdownloading asic_program.so ...done
5242024-01-17T05:12:36.276Zremote url: https://buildomat.eng.oxide.computer/public/file/oxidecomputer/sidecar-lite/release/e3ea4b495ba0a71801ded0776ae4bbd31df57e26/scadm.sha256.txt
5252024-01-17T05:12:43.340Zdownloading scadm ...done
5262024-01-17T05:12:43.419Zcreating port tfportrear0_0 for sc0_0
5272024-01-17T05:12:43.419Zcreating port tfportqsfp0_0 for sc0_1
5282024-01-17T05:12:43.419Zsoftnpu zone setup
5292024-01-17T05:12:43.419Zconfigure zone
5302024-01-17T05:12:43.419Zinstall zone
5312024-01-17T05:12:44.851Zboot zone
5322024-01-17T05:13:07.029Z[sidecar_softnpu] svcs milestone/network
5332024-01-17T05:13:07.546ZSTATE STIME FMRI
5342024-01-17T05:13:07.756Zonline 5:12:46 svc:/milestone/network:default
5352024-01-17T05:13:07.756Z[sidecar_softnpu] /softnpu/npu start
5362024-01-17T05:13:07.756Z
5372024-01-17T05:13:07.756Z+ /opt/oxide/work/tools/scrimlet/softnpu-init.sh
5382024-01-17T05:13:07.757Z+ GATEWAY_IP=10.151.2.100
5392024-01-17T05:13:07.757Z+ echo 'Using 10.151.2.100 as gateway ip'
5402024-01-17T05:13:07.757ZUsing 10.151.2.100 as gateway ip
5412024-01-17T05:13:07.757Z+ [[ ! -v GATEWAY_MAC ]]
5422024-01-17T05:13:07.757Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
5432024-01-17T05:13:07.757Z++ grep -c -E '^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$' /dev/fd/63
5442024-01-17T05:13:07.757Z+++ echo 18:c0:4d:0d:a0:2a
5452024-01-17T05:13:07.757Z+ COUNT=1
5462024-01-17T05:13:07.757Z+ [[ 1 -ne 1 ]]
5472024-01-17T05:13:07.757Z+ echo 'Using 18:c0:4d:0d:a0:2a as gateway mac'
5482024-01-17T05:13:07.757ZUsing 18:c0:4d:0d:a0:2a as gateway mac
5492024-01-17T05:13:07.757Z+ z_scadm add-arp-entry 10.151.2.100 18:c0:4d:0d:a0:2a
5502024-01-17T05:13:07.757Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone add-arp-entry 10.151.2.100 18:c0:4d:0d:a0:2a
5512024-01-17T05:13:07.757Z+ PXA_MAC=a8:e1:de:01:70:1d
5522024-01-17T05:13:07.757Z+ [[ -v PXA_START ]]
5532024-01-17T05:13:07.758Z+ z_scadm add-proxy-arp 10.151.2.164 10.151.2.227 a8:e1:de:01:70:1d
5542024-01-17T05:13:07.758Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone add-proxy-arp 10.151.2.164 10.151.2.227 a8:e1:de:01:70:1d
5552024-01-17T05:13:07.758Z+ z_scadm dump-state
5562024-01-17T05:13:07.758Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
5572024-01-17T05:13:07.760Zlocal v6:
5582024-01-17T05:13:07.760Zlocal v4:
5592024-01-17T05:13:07.760Zrouter v6:
5602024-01-17T05:13:07.760Zrouter v4_idx:
5612024-01-17T05:13:07.760Zrouter v4_routes:
5622024-01-17T05:13:07.760Zresolver v4:
5632024-01-17T05:13:07.760Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
5642024-01-17T05:13:07.760Zresolver v6:
5652024-01-17T05:13:07.760Znat v4:
5662024-01-17T05:13:07.761Znat_v6:
5672024-01-17T05:13:07.761Zport_mac:
5682024-01-17T05:13:07.761Zproxy arp:
5692024-01-17T05:13:07.761Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
5702024-01-17T05:13:07.761Z+ success 'softnpu zone exists'
5712024-01-17T05:13:07.761Z+ set +x
5722024-01-17T05:13:07.761Zsoftnpu zone exists
5732024-01-17T05:13:07.761Z
5742024-01-17T05:13:07.761Zreal 24.902541579
5752024-01-17T05:13:07.761Zuser 0.723041475
5762024-01-17T05:13:07.761Zsys 1.643496685
5772024-01-17T05:13:07.761Ztrap 0.000092325
5782024-01-17T05:13:07.761Ztflt 0.000180028
5792024-01-17T05:13:07.761Zdflt 0.000899433
5802024-01-17T05:13:07.761Zkflt 0.000000000
5812024-01-17T05:13:07.761Zlock 19:14.441996913
5822024-01-17T05:13:07.761Zslp 1:14.598357880
5832024-01-17T05:13:07.761Zlat 0.070113468
5842024-01-17T05:13:07.761Zstop 0.362461880
5852024-01-17T05:13:07.761Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml
5862024-01-17T05:13:07.761Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
5872024-01-17T05:13:07.762Z+ SILO_NAME=recovery
5882024-01-17T05:13:07.762Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
5892024-01-17T05:13:07.762Z+ EXTERNAL_DNS_DOMAIN=oxide.test
5902024-01-17T05:13:07.763Z+ sed -i~ '
5912024-01-17T05:13:07.763Z /^external_dns_ips/c\
5922024-01-17T05:13:07.763Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
5932024-01-17T05:13:07.763Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
5942024-01-17T05:13:07.763Z /^first/c\
5952024-01-17T05:13:07.763Zfirst = "10.151.2.164"
5962024-01-17T05:13:07.763Z /^last/c\
5972024-01-17T05:13:07.763Zlast = "10.151.2.173"
5982024-01-17T05:13:07.763Z }
5992024-01-17T05:13:07.763Z /^\[rack_network_config/,/^$/ {
6002024-01-17T05:13:07.763Z /^infra_ip_first/c\
6012024-01-17T05:13:07.763Zinfra_ip_first = "10.151.2.174"
6022024-01-17T05:13:07.763Z /^infra_ip_last/c\
6032024-01-17T05:13:07.763Zinfra_ip_last = "10.151.2.174"
6042024-01-17T05:13:07.763Z }
6052024-01-17T05:13:07.763Z /^\[\[rack_network_config.ports/,/^$/ {
6062024-01-17T05:13:07.763Z /^routes/c\
6072024-01-17T05:13:07.763Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
6082024-01-17T05:13:07.763Z /^addresses/c\
6092024-01-17T05:13:07.764Zaddresses = \["10.151.2.174/32"\]
6102024-01-17T05:13:07.764Z }
6112024-01-17T05:13:07.764Z' pkg/config-rss.toml
6122024-01-17T05:13:07.764Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
6132024-01-17T05:13:07.764Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
6142024-01-17T05:13:07.764Z+++ pkg/config-rss.toml Wed Jan 17 05:12:47 2024
6152024-01-17T05:13:07.764Z@@ -41,7 +41,7 @@
6162024-01-17T05:13:07.764Z # the DNS domain delegated to the rack by the customer. Each of these addresses
6172024-01-17T05:13:07.764Z # must be contained in one of the "internal services" IP Pool ranges listed
6182024-01-17T05:13:07.764Z # below.
6192024-01-17T05:13:07.764Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
6202024-01-17T05:13:07.764Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
6212024-01-17T05:13:07.764Z
6222024-01-17T05:13:07.764Z # Initial TLS certificates for the external API
6232024-01-17T05:13:07.764Z #
6242024-01-17T05:13:07.765Z@@ -77,8 +77,8 @@
6252024-01-17T05:13:07.765Z #
6262024-01-17T05:13:07.765Z # For more on this and what to put here, see docs/how-to-run.adoc.
6272024-01-17T05:13:07.765Z [[internal_services_ip_pool_ranges]]
6282024-01-17T05:13:07.765Z-first = "192.168.1.20"
6292024-01-17T05:13:07.765Z-last = "192.168.1.29"
6302024-01-17T05:13:07.765Z+first = "10.151.2.164"
6312024-01-17T05:13:07.766Z+last = "10.151.2.173"
6322024-01-17T05:13:07.766Z
6332024-01-17T05:13:07.766Z+ true
6342024-01-17T05:13:07.813Z # TODO - this configuration is subject to change going forward. Ultimately these
6352024-01-17T05:13:07.813Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml
6362024-01-17T05:13:07.813Z # parameters should be provided to the control plane via wicket, but we need to
6372024-01-17T05:13:07.813Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~
6382024-01-17T05:13:07.813Z+ TLS_NAME=recovery.sys.oxide.test
6392024-01-17T05:13:07.813Z@@ -92,8 +92,8 @@
6402024-01-17T05:13:07.813Z # A range of IP addresses used by Boundary Services on the external network. In
6412024-01-17T05:13:07.813Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
6422024-01-17T05:13:07.813Z # a real system, these would be addresses of the uplink ports on the Sidecar.
6432024-01-17T05:13:07.813Z # With softnpu, only one address is used.
6442024-01-17T05:13:07.814Z-infra_ip_first = "192.168.1.30"
6452024-01-17T05:13:07.814Z-infra_ip_last = "192.168.1.30"
6462024-01-17T05:13:07.814Z+infra_ip_first = "10.151.2.174"
6472024-01-17T05:13:07.814Z......+..+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.......+.......+.....+....+.....+...+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+.................................+...............+..+.+............+...+.....+....+...............+...+............+..+....+......+...+.......................+....+........+.+......+........+..........+.....+.......+.....+.........+.........+................+.........+...+.....+...+....+.....+......+......+....+.........+........+....+.....+.+............+............+...............+...+...+.........+...+.....+.............+...+..+...+......+.+.....+...+...............+........................+.+.....+....+...+..+...+....+.....+.+...............+........+.+...+.....+...+.......+.................+.............+...............+......+............+........+.+......+...+...........+.........+.........................+.........+..+.............+.....+.+...........+.+..+.......+...........+...+....+........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
6482024-01-17T05:13:07.814Z....+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+...................+..+.........+....+..+.+.........+...+..+....+..+.......+...+..+......+.+.....+...+.+......+......+...............+..+..........+........+.........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+...+.+......+.................+...+.........+...............+......+...................+...+..+....+..+..................+.+...+......+...........+..................+............+.............+..+.+.........+..............+.........+..........+...+...+.................+....+...+.....+..........+......+..+...+.........+...+....+...........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
6492024-01-17T05:13:07.814Z-----
6502024-01-17T05:13:07.814Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
6512024-01-17T05:13:07.814Z+infra_ip_last = "10.151.2.174"
6522024-01-17T05:13:07.814Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
6532024-01-17T05:13:07.814Z
6542024-01-17T05:13:07.814Z+ rmdir pkg
6552024-01-17T05:13:07.814Z # Configurations for BGP routers to run on the scrimlets.
6562024-01-17T05:13:07.817Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
6572024-01-17T05:13:07.817Z bgp = []
6582024-01-17T05:13:07.817Z+ pfexec mkdir -p /usr/oxide
6592024-01-17T05:13:07.817Z@@ -101,9 +101,9 @@
6602024-01-17T05:13:07.817Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v0.1.0
6612024-01-17T05:13:07.817Z+ pfexec chmod +x /usr/oxide/oxide
6622024-01-17T05:13:07.817Z # You can configure multiple uplinks by repeating the following stanza
6632024-01-17T05:13:07.817Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
6642024-01-17T05:13:07.817Z [[rack_network_config.ports]]
6652024-01-17T05:13:07.817Z # Routes associated with this port.
6662024-01-17T05:13:07.817Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
6672024-01-17T05:13:07.817Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
6682024-01-17T05:13:07.817Z # Addresses associated with this port.
6692024-01-17T05:13:07.818Z-addresses = ["192.168.1.30/32"]
6702024-01-17T05:13:07.818Z+addresses = ["10.151.2.174/32"]
6712024-01-17T05:13:07.818Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
6722024-01-17T05:13:07.818Z port = "qsfp0"
6732024-01-17T05:13:07.818Z # The speed of this port.
6742024-01-17T05:13:07.818Za pkg/config-rss.toml 7K
6752024-01-17T05:13:07.818Za pkg/initial-tls-cert.pem 2K
6762024-01-17T05:13:07.818Za pkg/initial-tls-key.pem 4K
6772024-01-17T05:13:07.818Z+ OMICRON_NO_UNINSTALL=1
6782024-01-17T05:13:07.818Z+ ptime -m pfexec ./target/release/omicron-package -t test install
6792024-01-17T05:13:07.818ZJan 17 05:13:07.868 DEBG target[test]: Target({"image": "standard", "machine": "non-gimlet", "rack-topology": "single-sled", "switch": "softnpu"})
6802024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/ntp.tar.gz, src: out/ntp.tar.gz
6812024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/internal_dns.tar.gz, src: out/internal-dns.tar.gz
6822024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/external_dns.tar.gz, src: out/external-dns.tar.gz
6832024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/crucible_pantry.tar.gz, src: out/crucible-pantry.tar.gz
6842024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/mgs.tar.gz, src: out/omicron-gateway-softnpu.tar.gz
6852024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/sled-agent.tar, src: out/omicron-sled-agent.tar
6862024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/cockroachdb.tar.gz, src: out/cockroachdb.tar.gz
6872024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/overlay.tar.gz, src: out/overlay.tar.gz
6882024-01-17T05:13:07.818ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/nexus.tar.gz, src: out/omicron-nexus.tar.gz
6892024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/clickhouse.tar.gz, src: out/clickhouse.tar.gz
6902024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/crucible.tar.gz, src: out/crucible.tar.gz
6912024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/switch.tar.gz, src: out/switch-softnpu.tar.gz
6922024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/oximeter.tar.gz, src: out/oximeter-collector.tar.gz
6932024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/propolis-server.tar.gz, src: out/propolis-server.tar.gz
6942024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/clickhouse_keeper.tar.gz, src: out/clickhouse_keeper.tar.gz
6952024-01-17T05:13:07.819ZJan 17 05:13:07.870 INFO Installing service, dst: /opt/oxide/mg-ddm.tar, src: out/mg-ddm-gz.tar
6962024-01-17T05:13:07.959ZJan 17 05:13:08.126 INFO Unpacking service tarball, service_path: /opt/oxide/mg-ddm, tar_path: /opt/oxide/mg-ddm.tar
6972024-01-17T05:13:08.008ZJan 17 05:13:08.162 INFO Unpacking service tarball, service_path: /opt/oxide/sled-agent, tar_path: /opt/oxide/sled-agent.tar
6982024-01-17T05:13:08.060ZJan 17 05:13:08.227 INFO Installing bootstrap service from /opt/oxide/sled-agent/pkg/manifest.xml
6992024-01-17T05:13:08.189Z
7002024-01-17T05:13:08.295Zreal 0.496795936
7012024-01-17T05:13:08.295Zuser 0.099343316
7022024-01-17T05:13:08.295Zsys 2.093628663
7032024-01-17T05:13:08.295Ztrap 0.000000570
7042024-01-17T05:13:08.296Ztflt 0.000026379
7052024-01-17T05:13:08.296Zdflt 0.000263521
7062024-01-17T05:13:08.296Zkflt 0.000012940
7072024-01-17T05:13:08.296Zlock 44.598903934
7082024-01-17T05:13:08.296Zslp 0.708543718
7092024-01-17T05:13:08.296Zlat 0.002965900
7102024-01-17T05:13:08.296Zstop 0.191829041
7112024-01-17T05:13:08.296Z+ retry=0
7122024-01-17T05:13:08.296Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7132024-01-17T05:13:08.296Z+ [[ 0 -gt 30 ]]
7142024-01-17T05:13:08.296Z+ sleep 1
7152024-01-17T05:13:09.203Z+ retry=1
7162024-01-17T05:13:09.203Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7172024-01-17T05:13:09.249Z+ [[ 1 -gt 30 ]]
7182024-01-17T05:13:09.249Z+ sleep 1
7192024-01-17T05:13:10.219Z+ retry=2
7202024-01-17T05:13:10.219Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7212024-01-17T05:13:10.248Z+ [[ 2 -gt 30 ]]
7222024-01-17T05:13:10.248Z+ sleep 1
7232024-01-17T05:13:11.233Z+ retry=3
7242024-01-17T05:13:11.234Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7252024-01-17T05:13:11.286Z+ [[ 3 -gt 30 ]]
7262024-01-17T05:13:11.286Z+ sleep 1
7272024-01-17T05:13:12.246Z+ retry=4
7282024-01-17T05:13:12.246Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7292024-01-17T05:13:37.417Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
7302024-01-17T05:13:37.445Zlocal v6:
7312024-01-17T05:13:37.445Zfd00:99::1
7322024-01-17T05:13:37.445Zfe80::b4d3:50ff:feb9:51db
7332024-01-17T05:13:37.445Zlocal v4:
7342024-01-17T05:13:37.445Z10.151.2.174
7352024-01-17T05:13:37.446Zrouter v6:
7362024-01-17T05:13:37.446Zfd00:1122:3344:1::/64 -> fe80::e023:90ff:feae:35b2 (1)
7372024-01-17T05:13:37.446Zfd00:1122:3344:2::/64 -> fe80::e023:90ff:feae:35b2 (1)
7382024-01-17T05:13:37.468Zfdb0:18c0:4d0d:a02a::/64 -> fe80::e023:90ff:feae:35b2 (1)
7392024-01-17T05:13:37.469Zfd00:1122:3344:3::/64 -> fe80::e023:90ff:feae:35b2 (1)
7402024-01-17T05:13:37.469Zfd00:1122:3344:101::/64 -> fe80::e023:90ff:feae:35b2 (1)
7412024-01-17T05:13:37.469Zrouter v4_idx:
7422024-01-17T05:13:37.469Z0.0.0.0/0 -> 0
7432024-01-17T05:13:37.469Zrouter v4_routes:
7442024-01-17T05:13:37.469Z0 -> 10.151.2.100 (2)
7452024-01-17T05:13:37.469Zresolver v4:
7462024-01-17T05:13:37.503Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
7472024-01-17T05:13:37.503Z10.151.2.174 -> b6:51:ff:e7:10:48
7482024-01-17T05:13:37.504Zresolver v6:
7492024-01-17T05:13:37.504Zfe80::b4d3:50ff:feb9:51db -> b6:d3:50:b9:51:db
7502024-01-17T05:13:37.504Zfe80::e023:90ff:feae:35b2 -> e2:23:90:ae:35:b2
7512024-01-17T05:13:37.504Znat v4:
7522024-01-17T05:13:37.504Z10.151.2.169 0/16383 -> fd00:1122:3344:101::1 100/a8:40:25:ff:8d:f7
7532024-01-17T05:13:37.504Znat_v6:
7542024-01-17T05:13:37.504Zport_mac:
7552024-01-17T05:13:37.504Z1000: a8:40:25:5e:34:68
7562024-01-17T05:13:37.504Z1: a8:40:25:5e:34:69
7572024-01-17T05:13:37.505Zproxy arp:
7582024-01-17T05:13:37.505Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
7592024-01-17T05:13:37.505Z+ retry=0
7602024-01-17T05:13:37.505Z+++ zoneadm list -n
7612024-01-17T05:13:37.505Z+++ grep oxz_ntp
7622024-01-17T05:13:37.505Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
7632024-01-17T05:13:37.505Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
7642024-01-17T05:13:37.505Z++ true
7652024-01-17T05:13:37.505Z+ [[ '' != online ]]
7662024-01-17T05:13:37.510Z+ [[ 0 -gt 60 ]]
7672024-01-17T05:13:37.510Z+ sleep 1
7682024-01-17T05:13:38.488Z+ retry=1
7692024-01-17T05:13:38.518Z+++ zoneadm list -n
7702024-01-17T05:13:38.518Z+++ grep oxz_ntp
7712024-01-17T05:13:38.518Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
7722024-01-17T05:13:38.518Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
7732024-01-17T05:13:38.518Z++ true
7742024-01-17T05:13:38.518Z+ [[ '' != online ]]
7752024-01-17T05:13:38.518Z+ [[ 1 -gt 60 ]]
7762024-01-17T05:13:38.518Z+ sleep 1
7772024-01-17T05:13:39.509Z+ retry=2
7782024-01-17T05:13:39.535Z+++ zoneadm list -n
7792024-01-17T05:13:39.535Z+++ grep oxz_ntp
7802024-01-17T05:13:39.535Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
7812024-01-17T05:13:39.561Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
7822024-01-17T05:13:39.561Z++ true
7832024-01-17T05:13:39.561Z+ [[ '' != online ]]
7842024-01-17T05:13:39.562Z+ [[ 2 -gt 60 ]]
7852024-01-17T05:13:39.562Z+ sleep 1
7862024-01-17T05:13:40.539Z+ retry=3
7872024-01-17T05:13:44.016Z+++ zoneadm list -n
7882024-01-17T05:13:44.016Z+++ grep oxz_ntp
7892024-01-17T05:13:44.016Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
7902024-01-17T05:13:44.016Z+ [[ disabled != online ]]
7912024-01-17T05:13:44.016Z+ [[ 3 -gt 60 ]]
7922024-01-17T05:13:44.016Z+ sleep 1
7932024-01-17T05:13:44.074Z+ retry=4
7942024-01-17T05:13:44.074Z+++ zoneadm list -n
7952024-01-17T05:13:44.074Z+++ grep oxz_ntp
7962024-01-17T05:13:44.074Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
7972024-01-17T05:13:44.075Z+ [[ disabled != online ]]
7982024-01-17T05:13:44.075Z+ [[ 4 -gt 60 ]]
7992024-01-17T05:13:44.075Z+ sleep 1
8002024-01-17T05:13:44.075Z+ retry=5
8012024-01-17T05:13:44.075Z+++ zoneadm list -n
8022024-01-17T05:13:44.075Z+++ grep oxz_ntp
8032024-01-17T05:13:44.075Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8042024-01-17T05:13:44.075Z+ [[ disabled != online ]]
8052024-01-17T05:13:44.075Z+ [[ 5 -gt 60 ]]
8062024-01-17T05:13:44.076Z+ sleep 1
8072024-01-17T05:13:44.083Z+ retry=6
8082024-01-17T05:13:44.083Z+++ zoneadm list -n
8092024-01-17T05:13:44.083Z+++ grep oxz_ntp
8102024-01-17T05:13:44.083Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8112024-01-17T05:13:44.083Z+ [[ disabled != online ]]
8122024-01-17T05:13:44.083Z+ [[ 6 -gt 60 ]]
8132024-01-17T05:13:44.084Z+ sleep 1
8142024-01-17T05:13:44.634Z+ retry=7
8152024-01-17T05:13:44.675Z+++ zoneadm list -n
8162024-01-17T05:13:44.675Z+++ grep oxz_ntp
8172024-01-17T05:13:44.675Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8182024-01-17T05:13:44.675Z+ [[ disabled != online ]]
8192024-01-17T05:13:44.675Z+ [[ 7 -gt 60 ]]
8202024-01-17T05:13:44.675Z+ sleep 1
8212024-01-17T05:13:45.682Z+ retry=8
8222024-01-17T05:13:45.720Z+++ zoneadm list -n
8232024-01-17T05:13:45.721Z+++ grep oxz_ntp
8242024-01-17T05:13:45.721Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8252024-01-17T05:13:45.721Z+ [[ disabled != online ]]
8262024-01-17T05:13:45.721Z+ [[ 8 -gt 60 ]]
8272024-01-17T05:13:45.721Z+ sleep 1
8282024-01-17T05:13:46.676Z+ retry=9
8292024-01-17T05:13:46.709Z+++ zoneadm list -n
8302024-01-17T05:13:46.709Z+++ grep oxz_ntp
8312024-01-17T05:13:46.710Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8322024-01-17T05:13:46.710Z+ [[ disabled != online ]]
8332024-01-17T05:13:46.710Z+ [[ 9 -gt 60 ]]
8342024-01-17T05:13:46.710Z+ sleep 1
8352024-01-17T05:13:47.695Z+ retry=10
8362024-01-17T05:13:47.736Z+++ zoneadm list -n
8372024-01-17T05:13:47.736Z+++ grep oxz_ntp
8382024-01-17T05:13:47.736Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8392024-01-17T05:13:47.736Z+ [[ disabled != online ]]
8402024-01-17T05:13:47.737Z+ [[ 10 -gt 60 ]]
8412024-01-17T05:13:47.737Z+ sleep 1
8422024-01-17T05:13:48.723Z+ retry=11
8432024-01-17T05:13:48.769Z+++ zoneadm list -n
8442024-01-17T05:13:48.769Z+++ grep oxz_ntp
8452024-01-17T05:13:48.769Z++ pfexec svcs -z oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e -Hostate oxide/ntp
8462024-01-17T05:13:48.770Z+ [[ online != online ]]
8472024-01-17T05:13:48.770Z+ echo 'Waited for chrony: 11s'
8482024-01-17T05:13:48.770ZWaited for chrony: 11s
8492024-01-17T05:13:48.770Z+ export RUST_BACKTRACE=1
8502024-01-17T05:13:48.770Z+ RUST_BACKTRACE=1
8512024-01-17T05:13:48.770Z+ export E2E_TLS_CERT IPPOOL_START IPPOOL_END
8522024-01-17T05:13:48.770Z++ ./tests/bootstrap
8532024-01-17T05:13:48.814ZUsing external DNS server at 10.151.2.164:53
8542024-01-17T05:13:48.815Z2024-01-17 05:13:48.938362819 UTC: attempting to log into API
8552024-01-17T05:14:03.814Z2024-01-17 05:14:03.742249271 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8562024-01-17T05:14:04.809Z2024-01-17 05:14:04.743336634 UTC: attempting to log into API
 
8572024-01-17T05:14:19.659ZError: No response to keepalive - Terminating session
 
8582024-01-17T05:14:19.842Z2024-01-17 05:14:19.776030987 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8592024-01-17T05:14:20.843Z2024-01-17 05:14:20.777054959 UTC: attempting to log into API
 
8602024-01-17T05:14:29.729ZError: No response de-activating SOL payload
8612024-01-17T05:14:30.826Z[SOL Session operational. Use ~? for help]
 
8622024-01-17T05:14:35.878Z2024-01-17 05:14:35.849785785 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8632024-01-17T05:14:36.888Z2024-01-17 05:14:36.853542602 UTC: attempting to log into API
8642024-01-17T05:14:51.913Z2024-01-17 05:14:51.900798230 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8652024-01-17T05:14:52.914Z2024-01-17 05:14:52.902820589 UTC: attempting to log into API
8662024-01-17T05:15:07.947Z2024-01-17 05:15:07.935572564 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8672024-01-17T05:15:08.954Z2024-01-17 05:15:08.936395961 UTC: attempting to log into API
8682024-01-17T05:15:23.983Z2024-01-17 05:15:23.969469438 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8692024-01-17T05:15:24.988Z2024-01-17 05:15:24.970551401 UTC: attempting to log into API
8702024-01-17T05:15:40.019Z2024-01-17 05:15:40.006840826 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
 
8712024-01-17T05:15:40.331ZJan 17 05:15:39 sock ip: ip: joining multicasts failed (4) on vopte2 - will use link layer broadcasts for multicast
 
8722024-01-17T05:15:41.019Z2024-01-17 05:15:41.007870857 UTC: attempting to log into API
 
8732024-01-17T05:15:45.889ZJan 17 05:15:45 sock ip: ip: joining multicasts failed (4) on vopte5 - will use link layer broadcasts for multicast
8742024-01-17T05:15:52.257ZJan 17 05:15:51 sock ip: ip: joining multicasts failed (4) on vopte1 - will use link layer broadcasts for multicast
 
8752024-01-17T05:15:56.053Z2024-01-17 05:15:56.041640658 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8762024-01-17T05:15:57.054Z2024-01-17 05:15:57.042884433 UTC: attempting to log into API
 
8772024-01-17T05:15:58.353ZJan 17 05:15:57 sock ip: ip: joining multicasts failed (4) on vopte3 - will use link layer broadcasts for multicast
 
8782024-01-17T05:16:12.086Z2024-01-17 05:16:12.075557561 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
8792024-01-17T05:16:13.087Z2024-01-17 05:16:13.076585672 UTC: attempting to log into API
8802024-01-17T05:16:18.864Z2024-01-17 05:16:18.853206459 UTC: login succeeded
8812024-01-17T05:16:18.905Zwaiting for nexus to come up...
8822024-01-17T05:16:18.964Zcreating IP pool... 10.151.2.175 - 10.151.2.227
8832024-01-17T05:16:19.169Zsetting up quotas...
8842024-01-17T05:16:19.224Zensuring datasets are ready...
8852024-01-17T05:16:21.890Zlet's roll.
8862024-01-17T05:16:21.923Z+ eval 'OXIDE_HOST=https://recovery.sys.oxide.test
8872024-01-17T05:16:21.923ZOXIDE_RESOLVE=recovery.sys.oxide.test:443:10.151.2.166
8882024-01-17T05:16:21.923ZOXIDE_TOKEN=oxide-token-659ce112ee9ea6ca1ebc5acf713b94f70ecfa473'
8892024-01-17T05:16:21.923Z++ OXIDE_HOST=https://recovery.sys.oxide.test
8902024-01-17T05:16:21.923Z++ OXIDE_RESOLVE=recovery.sys.oxide.test:443:10.151.2.166
8912024-01-17T05:16:21.923Z++ OXIDE_TOKEN=oxide-token-659ce112ee9ea6ca1ebc5acf713b94f70ecfa473
8922024-01-17T05:16:21.923Z+ export OXIDE_HOST OXIDE_TOKEN
8932024-01-17T05:16:21.923Z+ retry=0
8942024-01-17T05:16:21.923Z+ curl -sSf https://recovery.sys.oxide.test/v1/ping --resolve recovery.sys.oxide.test:443:10.151.2.166 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem
8952024-01-17T05:16:21.988Z+ /usr/oxide/oxide --resolve recovery.sys.oxide.test:443:10.151.2.166 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem project create --name images --description 'some images'
8962024-01-17T05:16:22.416Z{"status":"ok"}success
8972024-01-17T05:16:22.416ZProject {
8982024-01-17T05:16:22.416Z description: "some images",
8992024-01-17T05:16:22.416Z id: 02a5272b-c856-4609-965f-68e5f4629f91,
9002024-01-17T05:16:22.416Z name: Name(
9012024-01-17T05:16:22.417Z "images",
9022024-01-17T05:16:22.417Z ),
9032024-01-17T05:16:22.450Z time_created: 2024-01-17T05:16:22.065940Z,
9042024-01-17T05:16:22.450Z time_modified: 2024-01-17T05:16:22.065940Z,
9052024-01-17T05:16:22.450Z}
9062024-01-17T05:16:22.450Z+ /usr/oxide/oxide --resolve recovery.sys.oxide.test:443:10.151.2.166 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem disk import --path debian-11-genericcloud-amd64.raw --disk debian11-boot --project images --description 'debian 11 cloud image from distros' --snapshot debian11-snapshot --image debian11 --image-description 'debian 11 original base image' --image-os debian --image-version 11
 
9072024-01-17T05:17:19.391ZError: No response to keepalive - Terminating session
9082024-01-17T05:17:29.428ZError: No response de-activating SOL payload
9092024-01-17T05:17:36.267Z[SOL Session operational. Use ~? for help]
9102024-01-17T05:20:35.669ZError: No response to keepalive - Terminating session
9112024-01-17T05:20:45.457ZError: No response de-activating SOL payload
9122024-01-17T05:20:46.822Z[SOL Session operational. Use ~? for help]
 
9132024-01-17T05:21:55.726Zdone!
9142024-01-17T05:21:55.829Z+ /usr/oxide/oxide --resolve recovery.sys.oxide.test:443:10.151.2.166 --cacert /opt/oxide/sled-agent/pkg/initial-tls-cert.pem image promote --project images --image debian11
9152024-01-17T05:21:55.935Zsuccess
9162024-01-17T05:21:55.936ZImage {
9172024-01-17T05:21:55.968Z block_size: ByteCount(
9182024-01-17T05:21:55.968Z 512,
9192024-01-17T05:21:55.968Z ),
9202024-01-17T05:21:55.969Z description: "debian 11 original base image",
9212024-01-17T05:21:55.969Z digest: None,
9222024-01-17T05:21:55.969Z id: 95336ab9-0e9b-491c-aea1-55e80f9200a0,
9232024-01-17T05:21:55.969Z name: Name(
9242024-01-17T05:21:55.969Z "debian11",
9252024-01-17T05:21:55.969Z ),
9262024-01-17T05:21:55.969Z os: "debian",
9272024-01-17T05:21:55.969Z project_id: None,
9282024-01-17T05:21:55.969Z size: ByteCount(
9292024-01-17T05:21:55.969Z 2147483648,
9302024-01-17T05:21:55.969Z ),
9312024-01-17T05:21:55.969Z time_created: 2024-01-17T05:21:55.694221Z,
9322024-01-17T05:21:55.969Z time_modified: 2024-01-17T05:21:55.917443Z,
9332024-01-17T05:21:55.969Z url: None,
9342024-01-17T05:21:55.969Z version: "11",
9352024-01-17T05:21:55.969Z}
9362024-01-17T05:21:55.969Z+ rm ./tests/bootstrap
9372024-01-17T05:21:56.038Z+ for test_bin in tests/*
9382024-01-17T05:21:56.038Z+ ./tests/bootstrap-74528a6427e55c08
9392024-01-17T05:21:56.070Z
9402024-01-17T05:21:56.070Zrunning 0 tests
9412024-01-17T05:21:56.070Z
9422024-01-17T05:21:56.070Ztest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s
9432024-01-17T05:21:56.074Z
9442024-01-17T05:21:56.074Z+ for test_bin in tests/*
9452024-01-17T05:21:56.074Z+ ./tests/end_to_end_tests-346e854c34211b51
9462024-01-17T05:21:56.074Z
9472024-01-17T05:21:56.074Zrunning 2 tests
9482024-01-17T05:21:56.279Ztest no_spoof::no_spoof_here ... ok
9492024-01-17T05:22:32.858Ztest instance_launch::instance_launch ... ok
9502024-01-17T05:22:32.858Z
9512024-01-17T05:22:32.858Ztest result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 33.01s
9522024-01-17T05:22:32.858Z
9532024-01-17T05:22:32.858Z+ _exit_trap
9542024-01-17T05:22:32.858Z+ local status=0
9552024-01-17T05:22:32.859Z+ [[ 0 -eq 0 ]]
9562024-01-17T05:22:32.859Z+ exit 0
9572024-01-17T05:22:32.859Zprocess exited: duration 614441 ms, exit code 0
 
9582024-01-17T05:22:33.007Zfound 76 output files
9592024-01-17T05:22:33.007Zuploading: /var/svc/log/oxide-sled-agent:default.log (1538479 bytes)
9602024-01-17T05:22:34.932Zupload warning: file "/var/svc/log/oxide-sled-agent:default.log" changed size mid upload: 1538479 -> 1538877
9612024-01-17T05:22:36.437Zuploaded: /var/svc/log/oxide-sled-agent:default.log
9622024-01-17T05:22:36.437Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_67665777-44f3-4a40-af59-7b6abd64032e/root/var/svc/log/oxide-clickhouse:default.log (744975 bytes)
9632024-01-17T05:22:39.329Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_67665777-44f3-4a40-af59-7b6abd64032e/root/var/svc/log/oxide-clickhouse:default.log
9642024-01-17T05:22:39.329Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_67665777-44f3-4a40-af59-7b6abd64032e/root/var/svc/log/oxide-zone-network-setup:default.log (1708 bytes)
9652024-01-17T05:22:40.865Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_67665777-44f3-4a40-af59-7b6abd64032e/root/var/svc/log/oxide-zone-network-setup:default.log
9662024-01-17T05:22:40.932Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-cockroachdb:default.log (0 bytes)
9672024-01-17T05:22:41.556Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-cockroachdb:default.log
9682024-01-17T05:22:41.556Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-cockroachdb:default.log.0 (4406 bytes)
9692024-01-17T05:22:42.685Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-cockroachdb:default.log.0
9702024-01-17T05:22:42.685Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-zone-network-setup:default.log (0 bytes)
9712024-01-17T05:22:43.777Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-zone-network-setup:default.log
9722024-01-17T05:22:43.809Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-zone-network-setup:default.log.0 (1706 bytes)
9732024-01-17T05:22:44.902Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_df484553-aa35-414a-b8fa-1483088293f0/root/var/svc/log/oxide-zone-network-setup:default.log.0
9742024-01-17T05:22:44.902Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-agent:default.log (38472 bytes)
9752024-01-17T05:22:46.124Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-agent:default.log
9762024-01-17T05:22:46.124Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
9772024-01-17T05:22:47.266Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:default.log
9782024-01-17T05:22:47.298Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:downstairs-5db0682b-dd42-4282-9948-8aca8be47b49.log (3050 bytes)
9792024-01-17T05:22:48.397Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:downstairs-5db0682b-dd42-4282-9948-8aca8be47b49.log
9802024-01-17T05:22:48.397Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:downstairs-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0.log (5622 bytes)
9812024-01-17T05:22:49.552Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:downstairs-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0.log
9822024-01-17T05:22:49.552Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:downstairs-bb697436-c903-4a2c-a1fb-0df0c2a32c54.log (5513 bytes)
9832024-01-17T05:22:50.713Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:downstairs-bb697436-c903-4a2c-a1fb-0df0c2a32c54.log
9842024-01-17T05:22:50.750Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:snapshot-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0-f9283a43-16a4-41fc-928a-3b036fb10d47.log (5160 bytes)
9852024-01-17T05:22:51.839Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488/root/var/svc/log/oxide-crucible-downstairs:snapshot-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0-f9283a43-16a4-41fc-928a-3b036fb10d47.log
9862024-01-17T05:22:51.839Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_26a996cc-a435-48c2-9a79-3806ac033400/root/var/svc/log/oxide-external_dns:default.log (26941 bytes)
9872024-01-17T05:22:53.086Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_26a996cc-a435-48c2-9a79-3806ac033400/root/var/svc/log/oxide-external_dns:default.log
9882024-01-17T05:22:53.086Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_c6cbba3f-f37b-4e65-9d1f-84d8362bfe29/root/var/svc/log/oxide-internal_dns:default.log (26300 bytes)
9892024-01-17T05:22:54.332Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_c6cbba3f-f37b-4e65-9d1f-84d8362bfe29/root/var/svc/log/oxide-internal_dns:default.log
9902024-01-17T05:22:54.366Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_c6cbba3f-f37b-4e65-9d1f-84d8362bfe29/root/var/svc/log/oxide-internal_dns:default.log.0 (90219 bytes)
9912024-01-17T05:22:55.785Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_c6cbba3f-f37b-4e65-9d1f-84d8362bfe29/root/var/svc/log/oxide-internal_dns:default.log.0
9922024-01-17T05:22:55.820Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-cockroachdb:default.log (0 bytes)
9932024-01-17T05:22:56.882Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-cockroachdb:default.log
9942024-01-17T05:22:56.882Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-cockroachdb:default.log.0 (4407 bytes)
9952024-01-17T05:22:58.051Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-cockroachdb:default.log.0
9962024-01-17T05:22:58.052Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-zone-network-setup:default.log (0 bytes)
9972024-01-17T05:22:59.156Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-zone-network-setup:default.log
9982024-01-17T05:22:59.156Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-zone-network-setup:default.log.0 (1706 bytes)
9992024-01-17T05:23:00.287Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_675993fb-d671-491b-9aff-407f932ffe39/root/var/svc/log/oxide-zone-network-setup:default.log.0
10002024-01-17T05:23:00.320Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_4d871d5b-1a6b-4b0b-a1d2-384d0e5824e6/root/var/svc/log/oxide-crucible-agent:default.log (3436 bytes)
10012024-01-17T05:23:01.414Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_4d871d5b-1a6b-4b0b-a1d2-384d0e5824e6/root/var/svc/log/oxide-crucible-agent:default.log
10022024-01-17T05:23:01.414Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_4d871d5b-1a6b-4b0b-a1d2-384d0e5824e6/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10032024-01-17T05:23:02.543Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_4d871d5b-1a6b-4b0b-a1d2-384d0e5824e6/root/var/svc/log/oxide-crucible-downstairs:default.log
10042024-01-17T05:23:02.543Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_pantry_b4ab2f23-6d7b-47c7-89a7-9d4213ac66d9/root/var/svc/log/oxide-crucible-pantry:default.log (2337 bytes)
10052024-01-17T05:23:03.670Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_pantry_b4ab2f23-6d7b-47c7-89a7-9d4213ac66d9/root/var/svc/log/oxide-crucible-pantry:default.log
10062024-01-17T05:23:03.671Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_66ec4fc5-59dc-435c-8d66-1395a57518c7/root/var/svc/log/oxide-external_dns:default.log (26942 bytes)
10072024-01-17T05:23:04.856Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_66ec4fc5-59dc-435c-8d66-1395a57518c7/root/var/svc/log/oxide-external_dns:default.log
10082024-01-17T05:23:04.856Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_0315f0c8-33da-432d-8454-01e2239c74a9/root/var/svc/log/oxide-internal_dns:default.log (24610 bytes)
10092024-01-17T05:23:06.050Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_0315f0c8-33da-432d-8454-01e2239c74a9/root/var/svc/log/oxide-internal_dns:default.log
10102024-01-17T05:23:06.050Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_0315f0c8-33da-432d-8454-01e2239c74a9/root/var/svc/log/oxide-internal_dns:default.log.0 (88723 bytes)
10112024-01-17T05:23:07.341Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_0315f0c8-33da-432d-8454-01e2239c74a9/root/var/svc/log/oxide-internal_dns:default.log.0
10122024-01-17T05:23:07.341Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_7093a1e3-c528-4027-a6d6-3f71136002c2/root/var/svc/log/oxide-nexus:default.log (35684347 bytes)
10132024-01-17T05:23:13.627Zupload warning: file "/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_7093a1e3-c528-4027-a6d6-3f71136002c2/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 35684347 -> 35756888
10142024-01-17T05:23:14.723Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_7093a1e3-c528-4027-a6d6-3f71136002c2/root/var/svc/log/oxide-nexus:default.log
10152024-01-17T05:23:14.723Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/svc/log/oxide-ntp:default.log (139 bytes)
10162024-01-17T05:23:15.850Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/svc/log/oxide-ntp:default.log
10172024-01-17T05:23:15.850Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/svc/log/oxide-ntp:default.log.0 (1044 bytes)
10182024-01-17T05:23:16.990Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/svc/log/oxide-ntp:default.log.0
10192024-01-17T05:23:16.990Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_oximeter_7b00c2f4-fd12-4ca2-b0e9-668a436f1ff4/root/var/svc/log/oxide-oximeter:default.log (232977 bytes)
10202024-01-17T05:23:17.263Zupload warning: file "/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_oximeter_7b00c2f4-fd12-4ca2-b0e9-668a436f1ff4/root/var/svc/log/oxide-oximeter:default.log" changed size mid upload: 232977 -> 258471
10212024-01-17T05:23:18.362Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_oximeter_7b00c2f4-fd12-4ca2-b0e9-668a436f1ff4/root/var/svc/log/oxide-oximeter:default.log
10222024-01-17T05:23:18.362Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-cockroachdb:default.log (0 bytes)
10232024-01-17T05:23:19.866Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-cockroachdb:default.log
10242024-01-17T05:23:19.866Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-cockroachdb:default.log.0 (4381 bytes)
10252024-01-17T05:23:21.146Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-cockroachdb:default.log.0
10262024-01-17T05:23:21.146Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-zone-network-setup:default.log (0 bytes)
10272024-01-17T05:23:22.276Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-zone-network-setup:default.log
10282024-01-17T05:23:22.276Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-zone-network-setup:default.log.0 (1705 bytes)
10292024-01-17T05:23:23.406Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_cockroachdb_2045e16d-6027-4550-a9a7-0a4f6bc177a2/root/var/svc/log/oxide-zone-network-setup:default.log.0
10302024-01-17T05:23:23.438Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-agent:default.log (24369 bytes)
10312024-01-17T05:23:24.671Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-agent:default.log
10322024-01-17T05:23:24.671Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10332024-01-17T05:23:25.798Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-downstairs:default.log
10342024-01-17T05:23:25.803Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-downstairs:downstairs-723f20eb-cfe6-4fb1-96df-44080b244c54.log (5518 bytes)
10352024-01-17T05:23:27.364Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-downstairs:downstairs-723f20eb-cfe6-4fb1-96df-44080b244c54.log
10362024-01-17T05:23:27.364Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-downstairs:downstairs-abbf9d2b-d03f-42ce-9c23-25cad2247b32.log (3051 bytes)
10372024-01-17T05:23:28.489Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_77c8f6c4-f939-4c08-be7b-7a4a376dec47/root/var/svc/log/oxide-crucible-downstairs:downstairs-abbf9d2b-d03f-42ce-9c23-25cad2247b32.log
10382024-01-17T05:23:28.490Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_pantry_bdba0d50-c58d-4e1a-8b44-3f222418248a/root/var/svc/log/oxide-crucible-pantry:default.log (2333 bytes)
10392024-01-17T05:23:29.647Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_pantry_bdba0d50-c58d-4e1a-8b44-3f222418248a/root/var/svc/log/oxide-crucible-pantry:default.log
10402024-01-17T05:23:29.647Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_internal_dns_83aecd56-3aa0-4583-b062-8012bd046a6c/root/var/svc/log/oxide-internal_dns:default.log (24609 bytes)
10412024-01-17T05:23:29.838Zupload warning: file "/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_internal_dns_83aecd56-3aa0-4583-b062-8012bd046a6c/root/var/svc/log/oxide-internal_dns:default.log" changed size mid upload: 24609 -> 28365
10422024-01-17T05:23:30.932Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_internal_dns_83aecd56-3aa0-4583-b062-8012bd046a6c/root/var/svc/log/oxide-internal_dns:default.log
10432024-01-17T05:23:30.932Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_internal_dns_83aecd56-3aa0-4583-b062-8012bd046a6c/root/var/svc/log/oxide-internal_dns:default.log.0 (92475 bytes)
10442024-01-17T05:23:32.384Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_internal_dns_83aecd56-3aa0-4583-b062-8012bd046a6c/root/var/svc/log/oxide-internal_dns:default.log.0
10452024-01-17T05:23:32.385Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_nexus_96dcd7f1-25da-493e-8bd1-e426d93a043e/root/var/svc/log/oxide-nexus:default.log (877845 bytes)
10462024-01-17T05:23:33.900Zupload warning: file "/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_nexus_96dcd7f1-25da-493e-8bd1-e426d93a043e/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 877845 -> 949787
10472024-01-17T05:23:35.003Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_nexus_96dcd7f1-25da-493e-8bd1-e426d93a043e/root/var/svc/log/oxide-nexus:default.log
10482024-01-17T05:23:35.003Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-cockroachdb:default.log (0 bytes)
10492024-01-17T05:23:35.548ZError: No response to keepalive - Terminating session
10502024-01-17T05:23:36.517Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-cockroachdb:default.log
10512024-01-17T05:23:36.517Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-cockroachdb:default.log.0 (4406 bytes)
10522024-01-17T05:23:37.646Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-cockroachdb:default.log.0
10532024-01-17T05:23:37.646Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-zone-network-setup:default.log (0 bytes)
10542024-01-17T05:23:39.172Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-zone-network-setup:default.log
10552024-01-17T05:23:39.172Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-zone-network-setup:default.log.0 (1705 bytes)
10562024-01-17T05:23:40.304Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_cockroachdb_dd5e1f51-fe70-4201-9dc9-22622fd94a76/root/var/svc/log/oxide-zone-network-setup:default.log.0
10572024-01-17T05:23:40.304Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-agent:default.log (25240 bytes)
10582024-01-17T05:23:41.563Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-agent:default.log
10592024-01-17T05:23:41.563Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10602024-01-17T05:23:43.106Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-downstairs:default.log
10612024-01-17T05:23:43.106Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-downstairs:downstairs-18c16eb6-324c-4aa4-bea3-f47b03329196.log (5517 bytes)
10622024-01-17T05:23:44.260Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-downstairs:downstairs-18c16eb6-324c-4aa4-bea3-f47b03329196.log
10632024-01-17T05:23:44.292Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-downstairs:downstairs-362aee95-87d1-4117-add3-892bddb23454.log (2933 bytes)
10642024-01-17T05:23:45.430Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_2b55e815-a474-4fb3-a999-34836c6be239/root/var/svc/log/oxide-crucible-downstairs:downstairs-362aee95-87d1-4117-add3-892bddb23454.log
10652024-01-17T05:23:45.443Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_pantry_8a95a92a-a362-4a62-bbd8-4beb42536148/root/var/svc/log/oxide-crucible-pantry:default.log (1214673 bytes)
10662024-01-17T05:23:45.573ZError: No response de-activating SOL payload
10672024-01-17T05:23:46.687Z[SOL Session operational. Use ~? for help]
10682024-01-17T05:23:48.548Zupload warning: file "/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_pantry_8a95a92a-a362-4a62-bbd8-4beb42536148/root/var/svc/log/oxide-crucible-pantry:default.log" changed size mid upload: 1214673 -> 1246360
10692024-01-17T05:23:49.655Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_pantry_8a95a92a-a362-4a62-bbd8-4beb42536148/root/var/svc/log/oxide-crucible-pantry:default.log
10702024-01-17T05:23:49.656Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-cockroachdb:default.log (0 bytes)
10712024-01-17T05:23:50.752Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-cockroachdb:default.log
10722024-01-17T05:23:50.752Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-cockroachdb:default.log.0 (4406 bytes)
10732024-01-17T05:23:51.904Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-cockroachdb:default.log.0
10742024-01-17T05:23:51.907Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-zone-network-setup:default.log (0 bytes)
10752024-01-17T05:23:53.005Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-zone-network-setup:default.log
10762024-01-17T05:23:53.039Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-zone-network-setup:default.log.0 (1704 bytes)
10772024-01-17T05:23:54.136Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_8c846a2b-61b6-4ebf-a203-209cf9cf982c/root/var/svc/log/oxide-zone-network-setup:default.log.0
10782024-01-17T05:23:54.136Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_2c45b8ce-8081-40c8-aa96-2bf5dbda119c/root/var/svc/log/oxide-crucible-agent:default.log (9793 bytes)
10792024-01-17T05:23:55.297Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_2c45b8ce-8081-40c8-aa96-2bf5dbda119c/root/var/svc/log/oxide-crucible-agent:default.log
10802024-01-17T05:23:55.297Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_2c45b8ce-8081-40c8-aa96-2bf5dbda119c/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10812024-01-17T05:23:56.424Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_2c45b8ce-8081-40c8-aa96-2bf5dbda119c/root/var/svc/log/oxide-crucible-downstairs:default.log
10822024-01-17T05:23:56.425Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_2c45b8ce-8081-40c8-aa96-2bf5dbda119c/root/var/svc/log/oxide-crucible-downstairs:downstairs-16dcf881-2804-4e10-8a9e-f78df0b3d744.log (2933 bytes)
10832024-01-17T05:23:57.551Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_2c45b8ce-8081-40c8-aa96-2bf5dbda119c/root/var/svc/log/oxide-crucible-downstairs:downstairs-16dcf881-2804-4e10-8a9e-f78df0b3d744.log
10842024-01-17T05:23:57.551Zuploading: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_crucible_7dd8b277-bc77-40b7-97c9-dd3bba38812a/root/var/svc/log/oxide-crucible-agent:default.log (3436 bytes)
10852024-01-17T05:23:58.677Zuploaded: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_crucible_7dd8b277-bc77-40b7-97c9-dd3bba38812a/root/var/svc/log/oxide-crucible-agent:default.log
10862024-01-17T05:23:58.677Zuploading: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_crucible_7dd8b277-bc77-40b7-97c9-dd3bba38812a/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10872024-01-17T05:24:00.260Zuploaded: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_crucible_7dd8b277-bc77-40b7-97c9-dd3bba38812a/root/var/svc/log/oxide-crucible-downstairs:default.log
10882024-01-17T05:24:00.260Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-agent:default.log (28681 bytes)
10892024-01-17T05:24:01.505Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-agent:default.log
10902024-01-17T05:24:01.505Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10912024-01-17T05:24:02.632Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:default.log
10922024-01-17T05:24:02.632Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:downstairs-d5d024e6-a3a9-4896-9c9e-477bae821d8b.log (5622 bytes)
10932024-01-17T05:24:03.791Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:downstairs-d5d024e6-a3a9-4896-9c9e-477bae821d8b.log
10942024-01-17T05:24:03.824Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:downstairs-f5b1899e-db51-4534-b119-ead31d613d55.log (2929 bytes)
10952024-01-17T05:24:04.919Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:downstairs-f5b1899e-db51-4534-b119-ead31d613d55.log
10962024-01-17T05:24:04.919Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:snapshot-d5d024e6-a3a9-4896-9c9e-477bae821d8b-f9283a43-16a4-41fc-928a-3b036fb10d47.log (5157 bytes)
10972024-01-17T05:24:06.047Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_9f8dd614-d5d6-4fd1-b860-f9f6da7ae887/root/var/svc/log/oxide-crucible-downstairs:snapshot-d5d024e6-a3a9-4896-9c9e-477bae821d8b-f9283a43-16a4-41fc-928a-3b036fb10d47.log
10982024-01-17T05:24:06.048Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-agent:default.log (17122 bytes)
10992024-01-17T05:24:07.252Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-agent:default.log
11002024-01-17T05:24:07.253Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
11012024-01-17T05:24:08.392Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-downstairs:default.log
11022024-01-17T05:24:08.392Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-downstairs:downstairs-7c927793-3a8b-48de-a30d-5a982eaffa77.log (5621 bytes)
11032024-01-17T05:24:09.549Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-downstairs:downstairs-7c927793-3a8b-48de-a30d-5a982eaffa77.log
11042024-01-17T05:24:09.549Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-downstairs:snapshot-7c927793-3a8b-48de-a30d-5a982eaffa77-f9283a43-16a4-41fc-928a-3b036fb10d47.log (5158 bytes)
11052024-01-17T05:24:10.675Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_af6f21a7-2e6d-40ce-9f1b-426545843cc8/root/var/svc/log/oxide-crucible-downstairs:snapshot-7c927793-3a8b-48de-a30d-5a982eaffa77-f9283a43-16a4-41fc-928a-3b036fb10d47.log
11062024-01-17T05:24:10.675Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_900e3c4f-2a2f-4e92-93dc-a3b0db3e1052/root/var/svc/log/oxide-nexus:default.log (1136434 bytes)
11072024-01-17T05:24:12.919Zupload warning: file "/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_900e3c4f-2a2f-4e92-93dc-a3b0db3e1052/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 1136434 -> 1279136
11082024-01-17T05:24:14.019Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_900e3c4f-2a2f-4e92-93dc-a3b0db3e1052/root/var/svc/log/oxide-nexus:default.log
11092024-01-17T05:24:14.019Zuploading: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b81b0e31-9733-4a85-bcf2-e464c8edd2b8/root/var/svc/log/oxide-crucible-agent:default.log (13688 bytes)
11102024-01-17T05:24:15.221Zuploaded: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b81b0e31-9733-4a85-bcf2-e464c8edd2b8/root/var/svc/log/oxide-crucible-agent:default.log
11112024-01-17T05:24:15.255Zuploading: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b81b0e31-9733-4a85-bcf2-e464c8edd2b8/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
11122024-01-17T05:24:16.349Zuploaded: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b81b0e31-9733-4a85-bcf2-e464c8edd2b8/root/var/svc/log/oxide-crucible-downstairs:default.log
11132024-01-17T05:24:16.349Zuploading: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b81b0e31-9733-4a85-bcf2-e464c8edd2b8/root/var/svc/log/oxide-crucible-downstairs:downstairs-c67309af-988e-46e6-944b-0974b528be04.log (3047 bytes)
11142024-01-17T05:24:17.477Zuploaded: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_b81b0e31-9733-4a85-bcf2-e464c8edd2b8/root/var/svc/log/oxide-crucible-downstairs:downstairs-c67309af-988e-46e6-944b-0974b528be04.log
11152024-01-17T05:24:17.477Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/measurements.log (22468 bytes)
11162024-01-17T05:24:17.662Zupload warning: file "/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/measurements.log" changed size mid upload: 22468 -> 26304
11172024-01-17T05:24:18.756Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/measurements.log
11182024-01-17T05:24:18.757Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/statistics.log (14625 bytes)
11192024-01-17T05:24:18.902Zupload warning: file "/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/statistics.log" changed size mid upload: 14625 -> 17784
11202024-01-17T05:24:19.999Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/statistics.log
11212024-01-17T05:24:19.999Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/tracking.log (1985 bytes)
11222024-01-17T05:24:20.101Zupload warning: file "/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/tracking.log" changed size mid upload: 1985 -> 2249
11232024-01-17T05:24:21.217Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_ntp_979bbaed-cbb7-4f08-91db-16350521d50e/root/var/log/chrony/tracking.log