01HDHPCZC6N9WNNC3B8Y5EZEH6: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-sled-agent:default.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.25",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HDHPDQEM6G5PWW0FMTD429F5

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12023-10-24T21:26:41.429Zjob dependencies complete; ready to run (waiting for 50 m 33 s)
22023-10-24T21:27:26.817Zjob assigned to worker 01HDHSAESGW0992MXDNM1YQT6N (queued for 45 s)
32023-10-24T21:27:26.903Zhttp://172.20.0.5:9969/ipxe/sock... ok
42023-10-24T21:27:27.517Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52023-10-24T21:27:27.550Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62023-10-24T21:27:27.583Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72023-10-24T21:27:27.617Zhttp://172.20.0.5:9969/postboot/sock... ok
82023-10-24T21:27:27.651ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92023-10-24T21:27:27.684Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d1
102023-10-24T21:27:27.717ZOxide Helios Version helios-2.0.22241 64-bit
112023-10-24T21:27:27.751ZConfiguring devices.
122023-10-24T21:27:27.848Z##################################################
132023-10-24T21:27:27.885Z#### ############# ##
142023-10-24T21:27:27.918Z### ### ############ ##
152023-10-24T21:27:27.951Z## ### # ## ### ## Oxide Computer Company ##
162023-10-24T21:27:28.038Z## ## ## ### # ### ##
172023-10-24T21:27:28.108Z## # ### #### #### This Station Under ##
182023-10-24T21:27:28.140Z### ### #### # ### Computer Control ##
192023-10-24T21:27:28.174Z#### #### ### ## ##
202023-10-24T21:27:28.208Z##################################################
212023-10-24T21:27:28.250ZHostname: sock
222023-10-24T21:27:28.287ZLoading smf(7) serice descriptions:
232023-10-24T21:27:28.689Zdownloading input: /input/ci-tools/work/end-to-end-tests/bootstrap-49e31e75b930b622.gz
242023-10-24T21:27:30.542Zdownloaded input: /input/ci-tools/work/end-to-end-tests/bootstrap-49e31e75b930b622.gz
252023-10-24T21:27:30.570Zdownloading input: /input/ci-tools/work/end-to-end-tests/bootstrap.gz
262023-10-24T21:27:53.459Zdownloaded input: /input/ci-tools/work/end-to-end-tests/bootstrap.gz
272023-10-24T21:27:53.903Zdownloading input: /input/ci-tools/work/end-to-end-tests/end_to_end_tests-a91cf9af3457d280.gz
282023-10-24T21:28:21.872Zdownloaded input: /input/ci-tools/work/end-to-end-tests/end_to_end_tests-a91cf9af3457d280.gz
292023-10-24T21:28:21.897Zdownloading input: /input/ci-tools/work/caboose-util.gz
302023-10-24T21:28:22.913Zdownloaded input: /input/ci-tools/work/caboose-util.gz
312023-10-24T21:28:22.938Zdownloading input: /input/ci-tools/work/tufaceous.gz
322023-10-24T21:28:31.140Zdownloaded input: /input/ci-tools/work/tufaceous.gz
332023-10-24T21:28:31.166Zdownloading input: /input/package/work/version.txt
342023-10-24T21:28:31.214Zdownloaded input: /input/package/work/version.txt
352023-10-24T21:28:31.242Zdownloading input: /input/package/work/package.tar.gz
362023-10-24T21:28:50.373Zdownloaded input: /input/package/work/package.tar.gz
372023-10-24T21:28:50.398Zdownloading input: /input/package/work/global-zone-packages.tar.gz
382023-10-24T21:29:26.676Zdownloaded input: /input/package/work/global-zone-packages.tar.gz
392023-10-24T21:29:26.706Zdownloading input: /input/package/work/trampoline-global-zone-packages.tar.gz
402023-10-24T21:29:37.794Zdownloaded input: /input/package/work/trampoline-global-zone-packages.tar.gz
412023-10-24T21:29:37.822Zdownloading input: /input/package/work/zones/clickhouse.tar.gz
422023-10-24T21:30:19.999Zdownloaded input: /input/package/work/zones/clickhouse.tar.gz
432023-10-24T21:30:20.043Zdownloading input: /input/package/work/zones/clickhouse_keeper.tar.gz
442023-10-24T21:30:59.148Zdownloaded input: /input/package/work/zones/clickhouse_keeper.tar.gz
452023-10-24T21:30:59.173Zdownloading input: /input/package/work/zones/cockroachdb.tar.gz
462023-10-24T21:31:35.704Zdownloaded input: /input/package/work/zones/cockroachdb.tar.gz
472023-10-24T21:31:35.930Zdownloading input: /input/package/work/zones/crucible-pantry.tar.gz
482023-10-24T21:31:40.661Zdownloaded input: /input/package/work/zones/crucible-pantry.tar.gz
492023-10-24T21:31:40.704Zdownloading input: /input/package/work/zones/crucible.tar.gz
502023-10-24T21:31:49.995Zdownloaded input: /input/package/work/zones/crucible.tar.gz
512023-10-24T21:31:50.023Zdownloading input: /input/package/work/zones/external-dns.tar.gz
522023-10-24T21:31:54.916Zdownloaded input: /input/package/work/zones/external-dns.tar.gz
532023-10-24T21:31:54.943Zdownloading input: /input/package/work/zones/internal-dns.tar.gz
542023-10-24T21:32:00.362Zdownloaded input: /input/package/work/zones/internal-dns.tar.gz
552023-10-24T21:32:00.387Zdownloading input: /input/package/work/zones/ntp.tar.gz
562023-10-24T21:32:00.413Zdownloaded input: /input/package/work/zones/ntp.tar.gz
572023-10-24T21:32:00.438Zdownloading input: /input/package/work/zones/omicron-gateway-asic.tar.gz
582023-10-24T21:32:03.588Zdownloaded input: /input/package/work/zones/omicron-gateway-asic.tar.gz
592023-10-24T21:32:03.614Zdownloading input: /input/package/work/zones/omicron-gateway-softnpu.tar.gz
602023-10-24T21:32:06.988Zdownloaded input: /input/package/work/zones/omicron-gateway-softnpu.tar.gz
612023-10-24T21:32:07.017Zdownloading input: /input/package/work/zones/omicron-nexus-single-sled.tar.gz
622023-10-24T21:32:21.623Zdownloaded input: /input/package/work/zones/omicron-nexus-single-sled.tar.gz
632023-10-24T21:32:21.648Zdownloading input: /input/package/work/zones/omicron-nexus.tar.gz
642023-10-24T21:32:35.932Zdownloaded input: /input/package/work/zones/omicron-nexus.tar.gz
652023-10-24T21:32:35.957Zdownloading input: /input/package/work/zones/overlay.tar.gz
662023-10-24T21:32:36.000Zdownloaded input: /input/package/work/zones/overlay.tar.gz
672023-10-24T21:32:36.025Zdownloading input: /input/package/work/zones/oximeter-collector.tar.gz
682023-10-24T21:32:40.659Zdownloaded input: /input/package/work/zones/oximeter-collector.tar.gz
692023-10-24T21:32:40.683Zdownloading input: /input/package/work/zones/propolis-server.tar.gz
702023-10-24T21:32:53.170Zdownloaded input: /input/package/work/zones/propolis-server.tar.gz
712023-10-24T21:32:53.204Zdownloading input: /input/package/work/zones/switch-asic.tar.gz
722023-10-24T21:33:40.345Zdownloaded input: /input/package/work/zones/switch-asic.tar.gz
732023-10-24T21:33:40.369Zdownloading input: /input/package/work/zones/switch-softnpu.tar.gz
742023-10-24T21:34:14.402Zdownloaded input: /input/package/work/zones/switch-softnpu.tar.gz
 
752023-10-24T21:34:14.428Zstarting task 0: "setup"
762023-10-24T21:34:14.462Z++ uname -s
772023-10-24T21:34:14.486Z+ kern=SunOS
782023-10-24T21:34:14.511Z+ case "$kern" in
792023-10-24T21:34:14.535Z+ groupadd -g 12345 build
802023-10-24T21:34:14.559Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
812023-10-24T21:34:16.470Z+ zfs create -o mountpoint=/work rpool/work
822023-10-24T21:34:16.501Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
832023-10-24T21:34:16.530Z+ home_fs=zfs
842023-10-24T21:34:16.556Z+ [[ zfs == autofs ]]
852023-10-24T21:34:16.994Z+ mkdir -p /home/build
862023-10-24T21:34:17.030Z+ chown build:build /home/build /work
872023-10-24T21:34:18.501Z+ chmod 0700 /home/build /work
882023-10-24T21:34:18.526Zprocess exited: duration 4044 ms, exit code 0
 
892023-10-24T21:34:18.576Zstarting task 1: "authentication"
902023-10-24T21:34:18.646Zprocess exited: duration 44 ms, exit code 0
 
912023-10-24T21:34:18.695Zstarting task 2: "build"
922023-10-24T21:34:18.756Z+ trap _exit_trap EXIT
932023-10-24T21:34:18.781Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
942023-10-24T21:34:18.923Zsecpolicy_fs_linkdir: 0x55 = 0x55
952023-10-24T21:34:18.950Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
962023-10-24T21:34:18.975Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
972023-10-24T21:34:18.999Zsecpolicy_fs_linkdir+3: 0 = 0xe5
982023-10-24T21:34:19.024Zsecpolicy_fs_linkdir+4: 0 = 0xb8
992023-10-24T21:34:19.049Zsecpolicy_fs_linkdir+5: 0 = 0x1
1002023-10-24T21:34:19.073Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
1012023-10-24T21:34:19.097Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
1022023-10-24T21:34:19.122Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
1032023-10-24T21:34:19.146Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
1042023-10-24T21:34:19.171Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
1052023-10-24T21:34:19.196Z+ [[ -d /opt/oxide ]]
1062023-10-24T21:34:19.225Z+ pfexec mv /opt/oxide /opt/oxide-underneath
1072023-10-24T21:34:19.249Z+ pfexec mkdir /opt/oxide
1082023-10-24T21:34:19.277Z+ pfexec mount -F tmpfs -O swap /opt/oxide
1092023-10-24T21:34:19.301Z+ [[ -d /opt/oxide-underneath ]]
1102023-10-24T21:34:19.326Z+ cd /opt/oxide-underneath
1112023-10-24T21:34:19.357Z+ pfexec tar ceEp@/f - .
1122023-10-24T21:34:19.787Z+ cd /opt/oxide
1132023-10-24T21:34:19.817Z+ pfexec tar xveEp@/f -
1142023-10-24T21:34:19.853Zx ., 0 bytes, 0 tape blocks
1152023-10-24T21:34:19.879Zx ./opte, 0 bytes, 0 tape blocks
1162023-10-24T21:34:19.904Zx ./opte/bin, 0 bytes, 0 tape blocks
1172023-10-24T21:34:19.928Zx ./opte/bin/opteadm, 50714832 bytes, 99053 tape blocks
1182023-10-24T21:34:19.952Z+ pfexec rm -rf /opt/oxide-underneath
1192023-10-24T21:34:19.977Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
1202023-10-24T21:34:20.001Z+ pfexec mkdir /opt/oxide/work
1212023-10-24T21:34:20.026Z+ pfexec chown build:build /opt/oxide/work
1222023-10-24T21:34:20.050Z+ cd /opt/oxide/work
1232023-10-24T21:34:20.075Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
1242023-10-24T21:34:20.099Ztar: blocksize = 10
1252023-10-24T21:34:20.645ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
1262023-10-24T21:34:20.672Zx out/mg-ddm-gz.tar, 63638528 bytes, 124294 tape blocks
1272023-10-24T21:34:20.699Zx out/omicron-sled-agent.tar, 115460608 bytes, 225509 tape blocks
1282023-10-24T21:34:20.732Zx out/target/test, 75 bytes, 1 tape blocks
1292023-10-24T21:34:20.769Zx out/npuzone/npuzone, 37381232 bytes, 73011 tape blocks
1302023-10-24T21:34:20.794Zx package-manifest.toml, 22037 bytes, 44 tape blocks
1312023-10-24T21:34:20.819Zx smf/sled-agent/non-gimlet/config.toml, 3097 bytes, 7 tape blocks
1322023-10-24T21:34:20.844Zx target/release/omicron-package, 38196032 bytes, 74602 tape blocks
1332023-10-24T21:34:20.869Zx tools/create_virtual_hardware.sh, 2986 bytes, 6 tape blocks
1342023-10-24T21:34:20.894Zx tools/virtual_hardware.sh, 3408 bytes, 7 tape blocks
1352023-10-24T21:34:20.919Zx tools/scrimlet/create-softnpu-zone.sh, 619 bytes, 2 tape blocks
1362023-10-24T21:34:20.944Zx tools/scrimlet/destroy-softnpu-zone.sh, 195 bytes, 1 tape blocks
1372023-10-24T21:34:20.973Zx tools/scrimlet/softnpu-init.sh, 2148 bytes, 5 tape blocks
1382023-10-24T21:34:21.002Zx tools/scrimlet/softnpu-zone.txt, 330 bytes, 1 tape blocks
1392023-10-24T21:34:21.031Zx tools/scrimlet/softnpu.toml, 171 bytes, 1 tape blocks
1402023-10-24T21:34:21.061Z
1412023-10-24T21:34:21.088Zreal 1.562476534
1422023-10-24T21:34:21.112Zuser 1.409030406
1432023-10-24T21:34:21.138Zsys 0.290762947
1442023-10-24T21:34:21.165Ztrap 0.000051480
1452023-10-24T21:34:21.191Ztflt 0.000000000
1462023-10-24T21:34:21.219Zdflt 0.000000000
1472023-10-24T21:34:21.247Zkflt 0.000000000
1482023-10-24T21:34:21.272Zlock 0.000000000
1492023-10-24T21:34:21.297Zslp 1.381024326
1502023-10-24T21:34:21.321Zlat 0.041927050
1512023-10-24T21:34:21.394Zstop 0.000121501
1522023-10-24T21:34:21.419Z+ cp /input/package/work/zones/clickhouse_keeper.tar.gz /input/package/work/zones/clickhouse.tar.gz /input/package/work/zones/cockroachdb.tar.gz /input/package/work/zones/crucible-pantry.tar.gz /input/package/work/zones/crucible.tar.gz /input/package/work/zones/external-dns.tar.gz /input/package/work/zones/internal-dns.tar.gz /input/package/work/zones/ntp.tar.gz /input/package/work/zones/omicron-gateway-asic.tar.gz /input/package/work/zones/omicron-gateway-softnpu.tar.gz /input/package/work/zones/omicron-nexus-single-sled.tar.gz /input/package/work/zones/omicron-nexus.tar.gz /input/package/work/zones/overlay.tar.gz /input/package/work/zones/oximeter-collector.tar.gz /input/package/work/zones/propolis-server.tar.gz /input/package/work/zones/switch-asic.tar.gz /input/package/work/zones/switch-softnpu.tar.gz out/
1532023-10-24T21:34:21.672Z+ mv out/omicron-nexus-single-sled.tar.gz out/omicron-nexus.tar.gz
1542023-10-24T21:34:21.697Z+ mkdir tests
1552023-10-24T21:34:21.724Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1562023-10-24T21:34:21.749Z+ ptime -m gunzip
1572023-10-24T21:34:21.773Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap-49e31e75b930b622
1582023-10-24T21:34:21.833Z
1592023-10-24T21:34:21.858Zreal 0.145560241
1602023-10-24T21:34:21.882Zuser 0.131879336
1612023-10-24T21:34:21.906Zsys 0.013436088
1622023-10-24T21:34:21.931Ztrap 0.000009200
1632023-10-24T21:34:21.955Ztflt 0.000000000
1642023-10-24T21:34:21.980Zdflt 0.000000000
1652023-10-24T21:34:22.007Zkflt 0.000000000
1662023-10-24T21:34:22.037Zlock 0.000000000
1672023-10-24T21:34:22.067Zslp 0.000027130
1682023-10-24T21:34:22.100Zlat 0.000021640
1692023-10-24T21:34:22.124Zstop 0.000124972
1702023-10-24T21:34:22.150Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap-49e31e75b930b622
1712023-10-24T21:34:22.175Z+ chmod a+x tests/bootstrap-49e31e75b930b622
1722023-10-24T21:34:22.207Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1732023-10-24T21:34:22.670Z+ ptime -m gunzip
1742023-10-24T21:34:22.695Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap
1752023-10-24T21:34:24.102Z
1762023-10-24T21:34:24.132Zreal 2.252623577
1772023-10-24T21:34:24.164Zuser 2.109575523
1782023-10-24T21:34:24.196Zsys 0.142675496
1792023-10-24T21:34:24.238Ztrap 0.000155874
1802023-10-24T21:34:24.269Ztflt 0.000000000
1812023-10-24T21:34:24.301Zdflt 0.000000000
1822023-10-24T21:34:24.338Zkflt 0.000000000
1832023-10-24T21:34:24.369Zlock 0.000000000
1842023-10-24T21:34:24.405Zslp 0.000022230
1852023-10-24T21:34:24.444Zlat 0.000019350
1862023-10-24T21:34:24.476Zstop 0.000109161
1872023-10-24T21:34:24.502Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap
1882023-10-24T21:34:24.529Z+ chmod a+x tests/bootstrap
1892023-10-24T21:34:24.556Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1902023-10-24T21:34:24.583Z+ ptime -m gunzip
1912023-10-24T21:34:24.626Z++ basename /input/ci-tools/work/end-to-end-tests/end_to_end_tests-a91cf9af3457d280
1922023-10-24T21:34:26.690Z
1932023-10-24T21:34:27.131Zreal 2.541625753
1942023-10-24T21:34:27.159Zuser 2.398954326
1952023-10-24T21:34:27.192Zsys 0.142258799
1962023-10-24T21:34:27.220Ztrap 0.000194372
1972023-10-24T21:34:27.252Ztflt 0.000000000
1982023-10-24T21:34:27.283Zdflt 0.000000000
1992023-10-24T21:34:27.312Zkflt 0.000000000
2002023-10-24T21:34:27.336Zlock 0.000000000
2012023-10-24T21:34:27.361Zslp 0.000022840
2022023-10-24T21:34:27.388Zlat 0.000017250
2032023-10-24T21:34:27.413Zstop 0.000112572
2042023-10-24T21:34:27.439Z++ basename /input/ci-tools/work/end-to-end-tests/end_to_end_tests-a91cf9af3457d280
2052023-10-24T21:34:27.463Z+ chmod a+x tests/end_to_end_tests-a91cf9af3457d280
2062023-10-24T21:34:27.488Z+ bmat address ls
2072023-10-24T21:34:27.512ZNAME CIDR FIRST LAST COUNT
2082023-10-24T21:34:27.536Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
2092023-10-24T21:34:27.561Z++ bmat address ls -f extra -Ho first,last,count
2102023-10-24T21:34:27.590Z+ set -- 10.151.2.164 10.151.2.227 64
2112023-10-24T21:34:27.618Z+ EXTRA_IP_START=10.151.2.164
2122023-10-24T21:34:27.648Z+ EXTRA_IP_END=10.151.2.227
2132023-10-24T21:34:27.674Z+ EXTRA_IP_COUNT=64
2142023-10-24T21:34:27.713Z+ (( EXTRA_IP_COUNT >= 32 ))
2152023-10-24T21:34:27.740Z+ EXTRA_IP_BASE=10.151.2
2162023-10-24T21:34:27.769Z+ EXTRA_IP_FOCTET=164
2172023-10-24T21:34:27.795Z+ EXTRA_IP_LOCTET=227
2182023-10-24T21:34:27.819Z+ SERVICE_IP_POOL_START=10.151.2.164
2192023-10-24T21:34:27.848Z+ SERVICE_IP_POOL_END=10.151.2.173
2202023-10-24T21:34:27.873Z+ DNS_IP1=10.151.2.164
2212023-10-24T21:34:27.899Z+ DNS_IP2=10.151.2.165
2222023-10-24T21:34:27.927Z+ UPLINK_IP=10.151.2.174
2232023-10-24T21:34:27.956Z+ IPPOOL_START=10.151.2.175
2242023-10-24T21:34:27.981Z+ IPPOOL_END=10.151.2.227
2252023-10-24T21:34:28.009Z++ ipadm show-addr -po type,addr
2262023-10-24T21:34:28.035Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
2272023-10-24T21:34:28.063Z+ GATEWAY_IP=10.151.2.100
2282023-10-24T21:34:28.096Z+ [[ -n 10.151.2.100 ]]
2292023-10-24T21:34:28.134Z+ ping -s 10.151.2.100 56 1
2302023-10-24T21:34:28.163ZPING 10.151.2.100: 56 data bytes
2312023-10-24T21:34:28.187Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.012 ms
2322023-10-24T21:34:28.212Z
2332023-10-24T21:34:28.237Z----10.151.2.100 PING Statistics----
2342023-10-24T21:34:28.262Z1 packets transmitted, 1 packets received, 0% packet loss
2352023-10-24T21:34:28.287Zround-trip (ms) min/avg/max/stddev = 0.012/0.012/0.012/-nan
2362023-10-24T21:34:28.312Z++ arp -an
2372023-10-24T21:34:28.337Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
2382023-10-24T21:34:28.361Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
2392023-10-24T21:34:28.386Z+ routeadm -e ipv4-forwarding -u
2402023-10-24T21:34:28.411Z+ PXA_START=10.151.2.164
2412023-10-24T21:34:28.436Z+ PXA_END=10.151.2.227
2422023-10-24T21:34:28.461Z+ export GATEWAY_IP GATEWAY_MAC PXA_START PXA_END
2432023-10-24T21:34:28.486Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
2442023-10-24T21:34:28.510Z+ ZPOOL_VDEV_DIR=/scratch
2452023-10-24T21:34:28.535Z+ ptime -m pfexec ./tools/create_virtual_hardware.sh
2462023-10-24T21:34:28.560Z+++ dirname ./tools/create_virtual_hardware.sh
2472023-10-24T21:34:28.589Z++ cd ./tools
2482023-10-24T21:34:28.625Z++ pwd
2492023-10-24T21:34:28.653Z+ SOURCE_DIR=/opt/oxide/work/tools
2502023-10-24T21:34:28.680Z+ OMICRON_TOP=/opt/oxide/work/tools/..
2512023-10-24T21:34:28.705Z+ . /opt/oxide/work/tools/virtual_hardware.sh
2522023-10-24T21:34:28.769Z++ MARKER=/etc/opt/oxide/NO_INSTALL
2532023-10-24T21:34:28.794Z++ [[ -f /etc/opt/oxide/NO_INSTALL ]]
2542023-10-24T21:34:28.818Z++ dladm show-phys -p -o LINK
2552023-10-24T21:34:28.845Z++ head -1
2562023-10-24T21:34:28.869Z+ PHYSICAL_LINK=igb0
2572023-10-24T21:34:28.894Z+ echo 'Using igb0 as physical link'
2582023-10-24T21:34:28.920ZUsing igb0 as physical link
2592023-10-24T21:34:28.948Z+ ensure_run_as_root
2602023-10-24T21:34:28.978Z++ id -u
2612023-10-24T21:34:29.010Z+ [[ 0 -ne 0 ]]
2622023-10-24T21:34:29.043Z+ ensure_zpools
2632023-10-24T21:34:29.076Z+ ZPOOL_TYPES=('oxp_' 'oxi_')
2642023-10-24T21:34:29.118Z+ for ZPOOL_TYPE in "${ZPOOL_TYPES[@]}"
2652023-10-24T21:34:29.154Z+ readarray -t ZPOOLS
2662023-10-24T21:34:29.183Z++ grep '"oxp_' /opt/oxide/work/tools/../smf/sled-agent/non-gimlet/config.toml
2672023-10-24T21:34:29.211Z++ sed 's/[ ",]//g'
2682023-10-24T21:34:29.241Z+ for ZPOOL in "${ZPOOLS[@]}"
2692023-10-24T21:34:29.273Z+ echo 'Zpool: [oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b]'
2702023-10-24T21:34:29.299ZZpool: [oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b]
2712023-10-24T21:34:29.327Z+ VDEV_PATH=/scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
2722023-10-24T21:34:29.353Z+ [[ -f /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev ]]
2732023-10-24T21:34:29.379Z+ dd if=/dev/zero of=/scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev bs=1 count=0 seek=20G
2742023-10-24T21:34:29.404Z0+0 records in
2752023-10-24T21:34:29.434Z0+0 records out
2762023-10-24T21:34:29.467ZZFS vdev /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists
2772023-10-24T21:34:29.492Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
2782023-10-24T21:34:29.519ZZFS zpool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b exists
2792023-10-24T21:34:29.961Z+ success 'ZFS vdev /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists'
2802023-10-24T21:34:29.995ZZpool: [oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03]
2812023-10-24T21:34:30.024Z+ set +x
2822023-10-24T21:34:30.050ZZFS vdev /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
2832023-10-24T21:34:30.077Z++ zpool list -o name
2842023-10-24T21:34:30.103ZZFS zpool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
2852023-10-24T21:34:30.138Z++ grep oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
2862023-10-24T21:34:30.165ZZpool: [oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03]
2872023-10-24T21:34:30.191Z+ [[ -z '' ]]
2882023-10-24T21:34:30.225ZZFS vdev /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
2892023-10-24T21:34:30.250Z+ zpool create -f oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /scratch/oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
2902023-10-24T21:34:30.275ZZFS zpool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
2912023-10-24T21:34:30.299Z+ success 'ZFS zpool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b exists'
2922023-10-24T21:34:30.324ZZpool: [oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03]
2932023-10-24T21:34:30.348Z+ set +x
2942023-10-24T21:34:30.373ZZFS vdev /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
2952023-10-24T21:34:30.397Z+ for ZPOOL in "${ZPOOLS[@]}"
2962023-10-24T21:34:30.422ZZFS zpool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
2972023-10-24T21:34:30.448Z+ echo 'Zpool: [oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03]'
2982023-10-24T21:34:30.481ZZpool: [oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03]
2992023-10-24T21:34:30.509Z+ VDEV_PATH=/scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3002023-10-24T21:34:30.539ZZFS vdev /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists
3012023-10-24T21:34:31.012Z+ [[ -f /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3022023-10-24T21:34:31.040ZZFS zpool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 exists
3032023-10-24T21:34:31.073Z+ dd if=/dev/zero of=/scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3042023-10-24T21:34:31.103ZZpool: [oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930]
3052023-10-24T21:34:31.132Z0+0 records in
3062023-10-24T21:34:31.158ZZFS vdev /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev exists
3072023-10-24T21:34:31.182Z0+0 records out
3082023-10-24T21:34:31.207ZZFS zpool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 exists
3092023-10-24T21:34:31.231Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
3102023-10-24T21:34:31.257ZZpool: [oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60]
3112023-10-24T21:34:31.284Z+ success 'ZFS vdev /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3122023-10-24T21:34:31.311ZZFS vdev /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev exists
3132023-10-24T21:34:31.339Z+ set +x
3142023-10-24T21:34:31.368ZZFS zpool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 exists
3152023-10-24T21:34:31.397Z++ zpool list -o name
3162023-10-24T21:34:31.423ZZpool: [oxp_31bd71cd-4736-4a12-a387-9b74b050396f]
3172023-10-24T21:34:31.460Z++ grep oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
3182023-10-24T21:34:31.488ZZFS vdev /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev exists
3192023-10-24T21:34:31.521Z+ [[ -z '' ]]
3202023-10-24T21:34:31.553ZZFS zpool oxp_31bd71cd-4736-4a12-a387-9b74b050396f exists
3212023-10-24T21:34:31.585Z+ zpool create -f oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3222023-10-24T21:34:31.610ZZpool: [oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7]
3232023-10-24T21:34:31.635Z+ success 'ZFS zpool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3242023-10-24T21:34:31.660ZZFS vdev /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev exists
3252023-10-24T21:34:31.685Z+ set +x
3262023-10-24T21:34:31.714ZZFS zpool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 exists
3272023-10-24T21:34:31.751Z+ for ZPOOL in "${ZPOOLS[@]}"
3282023-10-24T21:34:31.779ZZpool: [oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b]
3292023-10-24T21:34:31.815Z+ echo 'Zpool: [oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03]'
3302023-10-24T21:34:31.840ZZFS vdev /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists
3312023-10-24T21:34:31.864Z+ VDEV_PATH=/scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3322023-10-24T21:34:31.895ZZFS zpool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b exists
3332023-10-24T21:34:31.928Z+ [[ -f /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3342023-10-24T21:34:31.953ZZpool: [oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b]
3352023-10-24T21:34:31.978Z+ dd if=/dev/zero of=/scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3362023-10-24T21:34:32.003ZZFS vdev /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists
3372023-10-24T21:34:32.029Z0+0 records in
3382023-10-24T21:34:32.054ZZFS zpool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b exists
3392023-10-24T21:34:32.079Z0+0 records out
3402023-10-24T21:34:32.104ZSimnet net0/sc0_0 exists
3412023-10-24T21:34:32.129Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
3422023-10-24T21:34:32.153ZSimnet net1/sc1_0 exists
3432023-10-24T21:34:32.586Z+ success 'ZFS vdev /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3442023-10-24T21:34:32.612ZVnic sc0_1 exists
3452023-10-24T21:34:32.640Z+ set +x
3462023-10-24T21:34:32.666Zremote url: https://buildomat.eng.oxide.computer/public/file/oxidecomputer/softnpu/image/c1c42398c82b0220c8b5fa3bfba9c7a3bcaa0943/softnpu.sha256.txt
3472023-10-24T21:34:32.698Z++ zpool list -o name
3482023-10-24T21:34:32.723Z++ grep oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
3492023-10-24T21:34:32.748Z+ [[ -z '' ]]
3502023-10-24T21:34:32.781Z+ zpool create -f oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3512023-10-24T21:34:32.810Z+ success 'ZFS zpool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3522023-10-24T21:34:32.868Z+ set +x
3532023-10-24T21:34:32.897Z+ for ZPOOL in "${ZPOOLS[@]}"
3542023-10-24T21:34:32.929Z+ echo 'Zpool: [oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03]'
3552023-10-24T21:34:32.959Z+ VDEV_PATH=/scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3562023-10-24T21:34:32.985Z+ [[ -f /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3572023-10-24T21:34:33.017Z+ dd if=/dev/zero of=/scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3582023-10-24T21:34:33.042Z0+0 records in
3592023-10-24T21:34:33.067Z0+0 records out
3602023-10-24T21:34:33.092Z0 bytes (0 iB) transferred in 0.000004 secs (0 iB/sec)
3612023-10-24T21:34:33.116Z+ success 'ZFS vdev /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3622023-10-24T21:34:33.145Z+ set +x
3632023-10-24T21:34:33.170Z++ zpool list -o name
3642023-10-24T21:34:33.196Z++ grep oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03
3652023-10-24T21:34:33.228Z+ [[ -z '' ]]
3662023-10-24T21:34:33.264Z+ zpool create -f oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3672023-10-24T21:34:33.292Z+ success 'ZFS zpool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3682023-10-24T21:34:33.317Z+ set +x
3692023-10-24T21:34:33.342Z+ for ZPOOL in "${ZPOOLS[@]}"
3702023-10-24T21:34:33.368Z+ echo 'Zpool: [oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03]'
3712023-10-24T21:34:33.393Z+ VDEV_PATH=/scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3722023-10-24T21:34:33.418Z+ [[ -f /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev ]]
3732023-10-24T21:34:33.443Z+ dd if=/dev/zero of=/scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev bs=1 count=0 seek=20G
3742023-10-24T21:34:33.468Z0+0 records in
3752023-10-24T21:34:33.493Z0+0 records out
3762023-10-24T21:34:33.517Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
3772023-10-24T21:34:33.543Z+ success 'ZFS vdev /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev exists'
3782023-10-24T21:34:33.571Z+ set +x
3792023-10-24T21:34:33.596Z++ zpool list -o name
3802023-10-24T21:34:33.621Z++ grep oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03
3812023-10-24T21:34:33.645Z+ [[ -z '' ]]
3822023-10-24T21:34:33.670Z+ zpool create -f oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 /scratch/oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03.vdev
3832023-10-24T21:34:33.695Z+ success 'ZFS zpool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 exists'
3842023-10-24T21:34:33.720Z+ set +x
3852023-10-24T21:34:33.745Z+ for ZPOOL in "${ZPOOLS[@]}"
3862023-10-24T21:34:33.772Z+ echo 'Zpool: [oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930]'
3872023-10-24T21:34:33.850Z+ VDEV_PATH=/scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev
3882023-10-24T21:34:33.893Z+ [[ -f /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev ]]
3892023-10-24T21:34:33.918Z+ dd if=/dev/zero of=/scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev bs=1 count=0 seek=20G
3902023-10-24T21:34:33.943Z0+0 records in
3912023-10-24T21:34:33.968Z0+0 records out
3922023-10-24T21:34:33.999Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
3932023-10-24T21:34:34.030Z+ success 'ZFS vdev /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev exists'
3942023-10-24T21:34:34.062Z+ set +x
3952023-10-24T21:34:34.093Z++ zpool list -o name
3962023-10-24T21:34:34.122Z++ grep oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930
3972023-10-24T21:34:34.152Z+ [[ -z '' ]]
3982023-10-24T21:34:34.178Z+ zpool create -f oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 /scratch/oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930.vdev
3992023-10-24T21:34:34.203Z+ success 'ZFS zpool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 exists'
4002023-10-24T21:34:34.227Z+ set +x
4012023-10-24T21:34:34.251Z+ for ZPOOL in "${ZPOOLS[@]}"
4022023-10-24T21:34:34.276Z+ echo 'Zpool: [oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60]'
4032023-10-24T21:34:34.301Z+ VDEV_PATH=/scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev
4042023-10-24T21:34:34.325Z+ [[ -f /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev ]]
4052023-10-24T21:34:34.355Z+ dd if=/dev/zero of=/scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev bs=1 count=0 seek=20G
4062023-10-24T21:34:34.380Z0+0 records in
4072023-10-24T21:34:34.404Z0+0 records out
4082023-10-24T21:34:34.429Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4092023-10-24T21:34:34.455Z+ success 'ZFS vdev /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev exists'
4102023-10-24T21:34:34.483Z+ set +x
4112023-10-24T21:34:34.520Z++ zpool list -o name
4122023-10-24T21:34:34.552Z++ grep oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60
4132023-10-24T21:34:34.580Z+ [[ -z '' ]]
4142023-10-24T21:34:34.605Z+ zpool create -f oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 /scratch/oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60.vdev
4152023-10-24T21:34:34.629Z+ success 'ZFS zpool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 exists'
4162023-10-24T21:34:34.654Z+ set +x
4172023-10-24T21:34:34.679Z+ for ZPOOL in "${ZPOOLS[@]}"
4182023-10-24T21:34:34.703Z+ echo 'Zpool: [oxp_31bd71cd-4736-4a12-a387-9b74b050396f]'
4192023-10-24T21:34:34.728Z+ VDEV_PATH=/scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev
4202023-10-24T21:34:34.753Z+ [[ -f /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev ]]
4212023-10-24T21:34:34.778Z+ dd if=/dev/zero of=/scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev bs=1 count=0 seek=20G
4222023-10-24T21:34:34.803Z0+0 records in
4232023-10-24T21:34:34.828Z0+0 records out
4242023-10-24T21:34:34.853Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4252023-10-24T21:34:34.878Z+ success 'ZFS vdev /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev exists'
4262023-10-24T21:34:34.919Z+ set +x
4272023-10-24T21:34:34.959Z++ zpool list -o name
4282023-10-24T21:34:35.433Z++ grep oxp_31bd71cd-4736-4a12-a387-9b74b050396f
4292023-10-24T21:34:35.865Z+ [[ -z '' ]]
4302023-10-24T21:34:35.897Z+ zpool create -f oxp_31bd71cd-4736-4a12-a387-9b74b050396f /scratch/oxp_31bd71cd-4736-4a12-a387-9b74b050396f.vdev
4312023-10-24T21:34:35.987Z+ success 'ZFS zpool oxp_31bd71cd-4736-4a12-a387-9b74b050396f exists'
4322023-10-24T21:34:36.017Z+ set +x
4332023-10-24T21:34:36.042Z+ for ZPOOL in "${ZPOOLS[@]}"
4342023-10-24T21:34:36.071Z+ echo 'Zpool: [oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7]'
4352023-10-24T21:34:36.096Z+ VDEV_PATH=/scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev
4362023-10-24T21:34:36.121Z+ [[ -f /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev ]]
4372023-10-24T21:34:36.146Z+ dd if=/dev/zero of=/scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev bs=1 count=0 seek=20G
4382023-10-24T21:34:36.170Z0+0 records in
4392023-10-24T21:34:36.195Z0+0 records out
4402023-10-24T21:34:36.221Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4412023-10-24T21:34:36.245Z+ success 'ZFS vdev /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev exists'
4422023-10-24T21:34:36.269Z+ set +x
4432023-10-24T21:34:36.297Z++ zpool list -o name
4442023-10-24T21:34:36.326Z++ grep oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7
4452023-10-24T21:34:36.361Z+ [[ -z '' ]]
4462023-10-24T21:34:36.480Z+ zpool create -f oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 /scratch/oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7.vdev
4472023-10-24T21:34:36.506Z+ success 'ZFS zpool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 exists'
4482023-10-24T21:34:36.530Z+ set +x
4492023-10-24T21:34:36.557Z+ for ZPOOL_TYPE in "${ZPOOL_TYPES[@]}"
4502023-10-24T21:34:36.582Z+ readarray -t ZPOOLS
4512023-10-24T21:34:36.607Z++ grep '"oxi_' /opt/oxide/work/tools/../smf/sled-agent/non-gimlet/config.toml
4522023-10-24T21:34:36.631Z++ sed 's/[ ",]//g'
4532023-10-24T21:34:36.656Z+ for ZPOOL in "${ZPOOLS[@]}"
4542023-10-24T21:34:36.680Z+ echo 'Zpool: [oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b]'
4552023-10-24T21:34:36.706Z+ VDEV_PATH=/scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4562023-10-24T21:34:36.731Z+ [[ -f /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev ]]
4572023-10-24T21:34:36.755Z+ dd if=/dev/zero of=/scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev bs=1 count=0 seek=20G
4582023-10-24T21:34:36.780Z0+0 records in
4592023-10-24T21:34:36.810Z0+0 records out
4602023-10-24T21:34:36.848Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4612023-10-24T21:34:36.875Z+ success 'ZFS vdev /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists'
4622023-10-24T21:34:36.902Z+ set +x
4632023-10-24T21:34:36.928Z++ zpool list -o name
4642023-10-24T21:34:36.958Z++ grep oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b
4652023-10-24T21:34:37.009Z+ [[ -z '' ]]
4662023-10-24T21:34:37.041Z+ zpool create -f oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b /scratch/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4672023-10-24T21:34:37.070Z+ success 'ZFS zpool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b exists'
4682023-10-24T21:34:37.098Z+ set +x
4692023-10-24T21:34:37.122Z+ for ZPOOL in "${ZPOOLS[@]}"
4702023-10-24T21:34:37.147Z+ echo 'Zpool: [oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b]'
4712023-10-24T21:34:37.173Z+ VDEV_PATH=/scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4722023-10-24T21:34:37.197Z+ [[ -f /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev ]]
4732023-10-24T21:34:37.221Z+ dd if=/dev/zero of=/scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev bs=1 count=0 seek=20G
4742023-10-24T21:34:37.247Z0+0 records in
4752023-10-24T21:34:37.293Z0+0 records out
4762023-10-24T21:34:37.319Z0 bytes (0 iB) transferred in 0.000003 secs (0 iB/sec)
4772023-10-24T21:34:37.352Z+ success 'ZFS vdev /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev exists'
4782023-10-24T21:34:37.377Z+ set +x
4792023-10-24T21:34:37.401Z++ zpool list -o name
4802023-10-24T21:34:37.426Z++ grep oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b
4812023-10-24T21:34:37.450Z+ [[ -z '' ]]
4822023-10-24T21:34:37.475Z+ zpool create -f oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b /scratch/oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b.vdev
4832023-10-24T21:34:37.504Z+ success 'ZFS zpool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b exists'
4842023-10-24T21:34:37.532Z+ set +x
4852023-10-24T21:34:37.557Z+ ensure_simulated_links igb0
4862023-10-24T21:34:37.581Z+ local PHYSICAL_LINK=igb0
4872023-10-24T21:34:37.606Z+ INDICES=("0" "1")
4882023-10-24T21:34:37.630Z+ for I in "${INDICES[@]}"
4892023-10-24T21:34:37.655Z++ get_simnet_name_if_exists net0
4902023-10-24T21:34:37.680Z++ dladm show-simnet -p -o LINK net0
4912023-10-24T21:34:37.716Z++ echo ''
4922023-10-24T21:34:37.748Z+ [[ -z '' ]]
4932023-10-24T21:34:37.773Z+ dladm create-simnet -t net0
4942023-10-24T21:34:37.797Z+ dladm create-simnet -t sc0_0
4952023-10-24T21:34:37.822Zdownloading softnpu ...done
4962023-10-24T21:34:37.846Z+ dladm modify-simnet -t -p net0 sc0_0
4972023-10-24T21:34:37.871Zremote url: https://buildomat.eng.oxide.computer/public/file/oxidecomputer/sidecar-lite/release/f397c5da66b2b54b0a288d89f0afaee00eb9a576/libsidecar_lite.so.sha256.txt
4982023-10-24T21:34:37.896Z+ dladm set-linkprop -p mtu=9000 sc0_0
4992023-10-24T21:34:37.920Z+ success 'Simnet net0/sc0_0 exists'
5002023-10-24T21:34:37.947Z+ set +x
5012023-10-24T21:34:37.974Z+ for I in "${INDICES[@]}"
5022023-10-24T21:34:38.006Z++ get_simnet_name_if_exists net1
5032023-10-24T21:34:38.052Z++ dladm show-simnet -p -o LINK net1
5042023-10-24T21:34:38.078Z++ echo ''
5052023-10-24T21:34:38.103Z+ [[ -z '' ]]
5062023-10-24T21:34:38.130Z+ dladm create-simnet -t net1
5072023-10-24T21:34:38.162Z+ dladm create-simnet -t sc1_0
5082023-10-24T21:34:38.189Z+ dladm modify-simnet -t -p net1 sc1_0
5092023-10-24T21:34:38.219Z+ dladm set-linkprop -p mtu=9000 sc1_0
5102023-10-24T21:34:38.244Z+ success 'Simnet net1/sc1_0 exists'
5112023-10-24T21:34:38.270Z+ set +x
5122023-10-24T21:34:38.294Z++ get_vnic_name_if_exists sc0_1
5132023-10-24T21:34:38.319Z++ dladm show-vnic -p -o LINK sc0_1
5142023-10-24T21:34:38.343Z++ echo ''
5152023-10-24T21:34:38.368Z+ [[ -z '' ]]
5162023-10-24T21:34:38.393Z+ dladm create-vnic -t sc0_1 -l igb0 -m a8:e1:de:01:70:1d
5172023-10-24T21:34:38.417Z+ [[ -v PROMISC_FILT_OFF ]]
5182023-10-24T21:34:38.452Z+ success 'Vnic sc0_1 exists'
5192023-10-24T21:34:38.891Z+ set +x
5202023-10-24T21:34:39.332Z+ warn_if_no_proxy_arp
5212023-10-24T21:34:39.363Z+ [[ -v PXA_START ]]
5222023-10-24T21:34:39.390Z+ [[ -v PXA_END ]]
5232023-10-24T21:34:39.415Z+ ensure_softnpu_zone
5242023-10-24T21:34:39.440Z+ zoneadm list
5252023-10-24T21:34:39.514Z+ grep -q sidecar_softnpu
5262023-10-24T21:34:39.539Z+ [[ -f out/npuzone/npuzone ]]
5272023-10-24T21:34:39.564Z+ out/npuzone/npuzone create sidecar --omicron-zone --ports sc0_0,tfportrear0_0 --ports sc0_1,tfportqsfp0_0 --sidecar-lite-branch omicron-tracking
5282023-10-24T21:34:44.500Zdownloading asic_program.so ...done
5292023-10-24T21:34:44.553Zremote url: https://buildomat.eng.oxide.computer/public/file/oxidecomputer/sidecar-lite/release/f397c5da66b2b54b0a288d89f0afaee00eb9a576/scadm.sha256.txt
5302023-10-24T21:34:50.337Zdownloading scadm ...done
5312023-10-24T21:34:50.363Zcreating port tfportrear0_0 for sc0_0
5322023-10-24T21:34:50.399Zcreating port tfportqsfp0_0 for sc0_1
5332023-10-24T21:34:50.437Zsoftnpu zone setup
5342023-10-24T21:34:50.493Zconfigure zone
5352023-10-24T21:34:50.518Zinstall zone
5362023-10-24T21:34:51.419Zboot zone
5372023-10-24T21:34:54.525Z[sidecar_softnpu] svcs milestone/network
5382023-10-24T21:34:54.551ZSTATE STIME FMRI
5392023-10-24T21:34:54.984Zonline 21:34:51 svc:/milestone/network:default
5402023-10-24T21:34:55.016Z[sidecar_softnpu] /softnpu/npu start
5412023-10-24T21:34:55.043Z
5422023-10-24T21:34:55.075Z+ /opt/oxide/work/tools/scrimlet/softnpu-init.sh
5432023-10-24T21:34:55.104Z+ GATEWAY_IP=10.151.2.100
5442023-10-24T21:34:55.129Z+ echo 'Using 10.151.2.100 as gateway ip'
5452023-10-24T21:34:55.163ZUsing 10.151.2.100 as gateway ip
5462023-10-24T21:34:55.191Z+ [[ ! -v GATEWAY_MAC ]]
5472023-10-24T21:34:55.216Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
5482023-10-24T21:34:55.241Z++ grep -c -E '^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$' /dev/fd/63
5492023-10-24T21:34:55.265Z+++ echo 18:c0:4d:0d:a0:2a
5502023-10-24T21:34:55.290Z+ COUNT=1
5512023-10-24T21:34:55.314Z+ [[ 1 -ne 1 ]]
5522023-10-24T21:34:55.341Z+ echo 'Using 18:c0:4d:0d:a0:2a as gateway mac'
5532023-10-24T21:34:55.365ZUsing 18:c0:4d:0d:a0:2a as gateway mac
5542023-10-24T21:34:55.391Z+ z_scadm add-arp-entry 10.151.2.100 18:c0:4d:0d:a0:2a
5552023-10-24T21:34:55.415Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone add-arp-entry 10.151.2.100 18:c0:4d:0d:a0:2a
5562023-10-24T21:34:55.439Z+ PXA_MAC=a8:e1:de:01:70:1d
5572023-10-24T21:34:55.464Z+ [[ -v PXA_START ]]
5582023-10-24T21:34:55.488Z+ z_scadm add-proxy-arp 10.151.2.164 10.151.2.227 a8:e1:de:01:70:1d
5592023-10-24T21:34:55.515Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone add-proxy-arp 10.151.2.164 10.151.2.227 a8:e1:de:01:70:1d
5602023-10-24T21:34:55.541Z+ z_scadm dump-state
5612023-10-24T21:34:55.581Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
5622023-10-24T21:34:55.610Zlocal v6:
5632023-10-24T21:34:55.634Zlocal v4:
5642023-10-24T21:34:55.659Zrouter v6:
5652023-10-24T21:34:55.684Zrouter v4:
5662023-10-24T21:34:55.708Zresolver v4:
5672023-10-24T21:34:55.734Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
5682023-10-24T21:34:55.759Zresolver v6:
5692023-10-24T21:34:55.783Znat v4:
5702023-10-24T21:34:55.808Znat_v6:
5712023-10-24T21:34:55.833Zport_mac:
5722023-10-24T21:34:55.857Zproxy arp:
5732023-10-24T21:34:55.883Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
5742023-10-24T21:34:55.909Z+ success 'softnpu zone exists'
5752023-10-24T21:34:55.933Z+ set +x
5762023-10-24T21:34:55.958Zsoftnpu zone exists
5772023-10-24T21:34:55.986Z
5782023-10-24T21:34:56.018Zreal 27.205418686
5792023-10-24T21:34:56.047Zuser 0.691699001
5802023-10-24T21:34:56.072Zsys 1.592945390
5812023-10-24T21:34:56.096Ztrap 0.000092626
5822023-10-24T21:34:56.121Ztflt 0.000145215
5832023-10-24T21:34:56.146Zdflt 0.000345072
5842023-10-24T21:34:56.170Zkflt 0.000000000
5852023-10-24T21:34:56.195Zlock 21:17.470214373
5862023-10-24T21:34:56.219Zslp 1:21.009701537
5872023-10-24T21:34:56.246Zlat 0.077702088
5882023-10-24T21:34:56.271Zstop 0.362101182
5892023-10-24T21:34:56.295Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml
5902023-10-24T21:34:56.319Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
5912023-10-24T21:34:56.344Z+ SILO_NAME=recovery
5922023-10-24T21:34:56.368Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
5932023-10-24T21:34:56.393Z+ EXTERNAL_DNS_DOMAIN=oxide.test
5942023-10-24T21:34:56.420Z+ sed -i~ '
5952023-10-24T21:34:56.446Z /^external_dns_ips/c\
5962023-10-24T21:34:56.471Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
5972023-10-24T21:34:56.496Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
5982023-10-24T21:34:56.521Z /^first/c\
5992023-10-24T21:34:56.546Zfirst = "10.151.2.164"
6002023-10-24T21:34:56.571Z /^last/c\
6012023-10-24T21:34:56.595Zlast = "10.151.2.173"
6022023-10-24T21:34:56.620Z }
6032023-10-24T21:34:56.644Z /^\[rack_network_config/,/^$/ {
6042023-10-24T21:34:56.669Z /^infra_ip_first/c\
6052023-10-24T21:34:56.693Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
6062023-10-24T21:34:56.717Zinfra_ip_first = "10.151.2.174"
6072023-10-24T21:34:56.759Z+++ pkg/config-rss.toml Tue Oct 24 21:34:53 2023
6082023-10-24T21:34:56.784Z /^infra_ip_last/c\
6092023-10-24T21:34:56.810Z@@ -41,7 +41,7 @@
6102023-10-24T21:34:56.835Zinfra_ip_last = "10.151.2.174"
6112023-10-24T21:34:56.861Z # the DNS domain delegated to the rack by the customer. Each of these addresses
6122023-10-24T21:34:57.295Z }
6132023-10-24T21:34:57.326Z # must be contained in one of the "internal services" IP Pool ranges listed
6142023-10-24T21:34:57.353Z /^\[\[rack_network_config.ports/,/^$/ {
6152023-10-24T21:34:57.378Z # below.
6162023-10-24T21:34:57.402Z /^routes/c\
6172023-10-24T21:34:57.428Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
6182023-10-24T21:34:57.452Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
6192023-10-24T21:34:57.476Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
6202023-10-24T21:34:57.501Z /^addresses/c\
6212023-10-24T21:34:57.525Z
6222023-10-24T21:34:57.549Zaddresses = \["10.151.2.174/32"\]
6232023-10-24T21:34:57.590Z # Initial TLS certificates for the external API
6242023-10-24T21:34:57.616Z }
6252023-10-24T21:34:57.642Z #
6262023-10-24T21:34:57.671Z' pkg/config-rss.toml
6272023-10-24T21:34:57.698Z@@ -77,8 +77,8 @@
6282023-10-24T21:34:58.164Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
6292023-10-24T21:34:58.195Z #
6302023-10-24T21:34:58.220Z+ true
6312023-10-24T21:34:58.245Z # For more on this and what to put here, see docs/how-to-run.adoc.
6322023-10-24T21:34:58.270Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml
6332023-10-24T21:34:58.295Z [[internal_services_ip_pool_ranges]]
6342023-10-24T21:34:58.320Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~
6352023-10-24T21:34:58.344Z-first = "192.168.1.20"
6362023-10-24T21:34:58.369Z+ TLS_NAME=recovery.sys.oxide.test
6372023-10-24T21:34:58.394Z-last = "192.168.1.29"
6382023-10-24T21:34:58.418Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
6392023-10-24T21:34:58.443Z+first = "10.151.2.164"
6402023-10-24T21:34:58.467Z........+.+..+.+..+.........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.......+.....+...............+.......+........+.+..+...+....+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+...........+.......+...+.........+...+........+.+......+.........+...+.................+.............+.....+.+...........+...+..................+...+.+...+..+....+.................+...+.........+.......+.........+.....+.........+..........+............+....................+...+.+.....+.........+.+.....+.+.....+.+...................................+.......+..+............................+......+............+..+.+.........+........+.......+...........+....+............+.....+...+.......+.........+..........................+....+..+...+.........+............+......+...............+....+.....+....+......+...+...........+.........+......+...................+...+..+......+.+.........+.................+............+.......+.....................+........+.+......+.........+...+..+.+..+......+.+...+......+.....+....+......+..+...........................+...+...+..........+.....................+........+..........+..+......+.......+.....+.......+........+.+..+..........+........+.+.....+....+............+...+..+.........................+.....+.....................................+.....+...+.......+.....+.+.....+......+...+......+.+...............+..................+............+........+.+..+.......+......+......+........+.+............+.........+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
6412023-10-24T21:34:58.491Z+last = "10.151.2.173"
6422023-10-24T21:34:58.516Z
6432023-10-24T21:34:58.540Z # TODO - this configuration is subject to change going forward. Ultimately these
6442023-10-24T21:34:58.565Z # parameters should be provided to the control plane via wicket, but we need to
6452023-10-24T21:34:58.596Z@@ -92,8 +92,8 @@
6462023-10-24T21:34:58.621Z # A range of IP addresses used by Boundary Services on the external network. In
6472023-10-24T21:34:58.646Z # a real system, these would be addresses of the uplink ports on the Sidecar.
6482023-10-24T21:34:58.671Z # With softnpu, only one address is used.
6492023-10-24T21:34:58.695Z-infra_ip_first = "192.168.1.30"
6502023-10-24T21:34:58.719Z-infra_ip_last = "192.168.1.30"
6512023-10-24T21:34:58.744Z+infra_ip_first = "10.151.2.174"
6522023-10-24T21:34:58.768Z+infra_ip_last = "10.151.2.174"
6532023-10-24T21:34:58.887Z
6542023-10-24T21:34:58.912Z # Configurations for BGP routers to run on the scrimlets.
6552023-10-24T21:34:58.936Z bgp = []
6562023-10-24T21:34:58.961Z@@ -101,9 +101,9 @@
6572023-10-24T21:34:58.986Z # You can configure multiple uplinks by repeating the following stanza
6582023-10-24T21:34:59.012Z [[rack_network_config.ports]]
6592023-10-24T21:34:59.047Z # Routes associated with this port.
6602023-10-24T21:34:59.083Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
6612023-10-24T21:34:59.108Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
6622023-10-24T21:34:59.133Z # Addresses associated with this port.
6632023-10-24T21:34:59.157Z-addresses = ["192.168.1.30/32"]
6642023-10-24T21:34:59.182Z......+.........+.........+.+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...........+..+.......+......+.....+.........+....+.........+..+......+...+.......+...+.........+...+...+...........+......+...+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+....+..+...+......+.+..............+....+..+...+...................+..+...+.+.....+......................+...+......+......+..+......+.+...+..+..........+...+...........+.......+..+.......+.....................+...............+..+.+.....+......+......+...+.+........+.......+...+........+.......+....................+...+...+.+....................+...+............+...+.+.........+........+...+......+.+...............+.....+...............+...+....+...............+...+........+.........+...+.......+.....+...+.........+.+........+...................+..+.........+.+...........+....+.........+......+.......................+......+......+.+............+...+..................+......+......+...+.........+......+......+.....+......+.........+...............+....+.....+......+...+.+......+...+.....+..........+.....+.+..+....+...+.................+.+........+.+........+.............+.....+.......+..+....+.................+...+....+.................+.+..+.+.....+......................+..+.......+...+..................+.....+............+...+...+.+........+.+......+......+...+............+..............+.+.....+...+...............+....+........+....+........+...+....+...+........+.......+..+...................+......+...+............+...+...+........+......+...+...+...............+....+......+..+...+...+..........+...........+.+.............................+..................+............+....+.....+.+...........+...+.........+.+........................+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
6652023-10-24T21:34:59.206Z+addresses = ["10.151.2.174/32"]
6662023-10-24T21:34:59.237Z-----
6672023-10-24T21:34:59.264Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
6682023-10-24T21:34:59.289Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
6692023-10-24T21:34:59.313Z port = "qsfp0"
6702023-10-24T21:34:59.338Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
6712023-10-24T21:34:59.362Z # The speed of this port.
6722023-10-24T21:34:59.386Z+ rmdir pkg
6732023-10-24T21:34:59.411Za pkg/config-rss.toml 7K
6742023-10-24T21:34:59.436Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
6752023-10-24T21:34:59.461Za pkg/initial-tls-cert.pem 2K
6762023-10-24T21:34:59.485Z+ pfexec mkdir -p /usr/oxide
6772023-10-24T21:34:59.519Za pkg/initial-tls-key.pem 4K
6782023-10-24T21:34:59.546Z+ pfexec rm -f /usr/oxide/tcpproxy
6792023-10-24T21:34:59.572Z+ pfexec curl -sSfL -o /usr/oxide/tcpproxy http://catacomb.eng.oxide.computer:12346/tcpproxy
6802023-10-24T21:34:59.596Z+ pfexec chmod +x /usr/oxide/tcpproxy
6812023-10-24T21:34:59.621Z+ pfexec rm -f /var/svc/manifest/site/tcpproxy.xml
6822023-10-24T21:34:59.647Z+ pfexec curl -sSfL -o /var/svc/manifest/site/tcpproxy.xml http://catacomb.eng.oxide.computer:12346/tcpproxy.xml
6832023-10-24T21:34:59.675Z+ pfexec svccfg import /var/svc/manifest/site/tcpproxy.xml
6842023-10-24T21:34:59.699Z+ OMICRON_NO_UNINSTALL=1
6852023-10-24T21:34:59.724Z+ ptime -m pfexec ./target/release/omicron-package -t test install
6862023-10-24T21:34:59.748ZOct 24 21:34:56.145 DEBG target[test]: Target({"image": "standard", "machine": "non-gimlet", "rack-topology": "single-sled", "switch": "softnpu"})
6872023-10-24T21:34:59.773ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/clickhouse.tar.gz, src: out/clickhouse.tar.gz
6882023-10-24T21:34:59.797ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/ntp.tar.gz, src: out/ntp.tar.gz
6892023-10-24T21:34:59.840ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/crucible_pantry.tar.gz, src: out/crucible-pantry.tar.gz
6902023-10-24T21:34:59.865ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/cockroachdb.tar.gz, src: out/cockroachdb.tar.gz
6912023-10-24T21:34:59.889ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/nexus.tar.gz, src: out/omicron-nexus.tar.gz
6922023-10-24T21:34:59.913ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/mgs.tar.gz, src: out/omicron-gateway-softnpu.tar.gz
6932023-10-24T21:34:59.941ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/crucible.tar.gz, src: out/crucible.tar.gz
6942023-10-24T21:34:59.967ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/oximeter.tar.gz, src: out/oximeter-collector.tar.gz
6952023-10-24T21:34:59.993ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/switch.tar.gz, src: out/switch-softnpu.tar.gz
6962023-10-24T21:35:00.018ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/external_dns.tar.gz, src: out/external-dns.tar.gz
6972023-10-24T21:35:00.042ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/internal_dns.tar.gz, src: out/internal-dns.tar.gz
6982023-10-24T21:35:00.068ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/mg-ddm.tar, src: out/mg-ddm-gz.tar
6992023-10-24T21:35:00.092ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/clickhouse_keeper.tar.gz, src: out/clickhouse_keeper.tar.gz
7002023-10-24T21:35:00.117ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/sled-agent.tar, src: out/omicron-sled-agent.tar
7012023-10-24T21:35:00.141ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/propolis-server.tar.gz, src: out/propolis-server.tar.gz
7022023-10-24T21:35:00.166ZOct 24 21:34:56.147 INFO Installing service, dst: /opt/oxide/overlay.tar.gz, src: out/overlay.tar.gz
7032023-10-24T21:35:00.197ZOct 24 21:34:56.398 INFO Unpacking service tarball, service_path: /opt/oxide/mg-ddm, tar_path: /opt/oxide/mg-ddm.tar
7042023-10-24T21:35:00.223ZOct 24 21:34:56.437 INFO Unpacking service tarball, service_path: /opt/oxide/sled-agent, tar_path: /opt/oxide/sled-agent.tar
7052023-10-24T21:35:00.247ZOct 24 21:34:56.511 INFO Installing bootstrap service from /opt/oxide/sled-agent/pkg/manifest.xml
7062023-10-24T21:35:00.272Z
7072023-10-24T21:35:00.297Zreal 0.506485618
7082023-10-24T21:35:00.322Zuser 0.096692955
7092023-10-24T21:35:00.350Zsys 1.958640364
7102023-10-24T21:35:00.380Ztrap 0.000000249
7112023-10-24T21:35:00.410Ztflt 0.000757676
7122023-10-24T21:35:00.439Zdflt 0.000495059
7132023-10-24T21:35:00.469Zkflt 0.000000000
7142023-10-24T21:35:00.498Zlock 45.695315348
7152023-10-24T21:35:00.530Zslp 0.724315041
7162023-10-24T21:35:00.557Zlat 0.002874886
7172023-10-24T21:35:00.588Zstop 0.206421386
7182023-10-24T21:35:00.619Z+ retry=0
7192023-10-24T21:35:00.650Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7202023-10-24T21:35:00.680Z+ [[ 0 -gt 30 ]]
7212023-10-24T21:35:00.715Z+ sleep 1
7222023-10-24T21:35:00.750Z+ retry=1
7232023-10-24T21:35:00.782Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7242023-10-24T21:35:00.813Z+ [[ 1 -gt 30 ]]
7252023-10-24T21:35:00.840Z+ sleep 1
7262023-10-24T21:35:00.877Z+ retry=2
7272023-10-24T21:35:00.902Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
7282023-10-24T21:35:24.721Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
7292023-10-24T21:35:24.746Zlocal v6:
7302023-10-24T21:35:24.770Zfe80::8b1:d3ff:fe15:2945
7312023-10-24T21:35:24.799Zfd00:99::1
7322023-10-24T21:35:24.827Zlocal v4:
7332023-10-24T21:35:24.858Z10.151.2.174
7342023-10-24T21:35:24.883Zrouter v6:
7352023-10-24T21:35:24.908Zfdb0:18c0:4d0d:a02a::/64 -> fe80::6c21:6cff:fe1c:c21c (1) vid=0
7362023-10-24T21:35:24.932Zfd00:1122:3344:101::/64 -> fe80::6c21:6cff:fe1c:c21c (1) vid=0
7372023-10-24T21:35:24.957Zrouter v4:
7382023-10-24T21:35:24.981Z0.0.0.0/0 -> 10.151.2.100 (2) vid=0
7392023-10-24T21:35:25.007Zresolver v4:
7402023-10-24T21:35:25.033Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
7412023-10-24T21:35:25.057Z10.151.2.174 -> 92:84:c1:07:3b:d4
7422023-10-24T21:35:25.081Zresolver v6:
7432023-10-24T21:35:25.106Zfe80::8b1:d3ff:fe15:2945 -> 0a:b1:d3:15:29:45
7442023-10-24T21:35:25.131Zfe80::6c21:6cff:fe1c:c21c -> 6e:21:6c:1c:c2:1c
7452023-10-24T21:35:25.155Znat v4:
7462023-10-24T21:35:25.179Znat_v6:
7472023-10-24T21:35:25.204Zport_mac:
7482023-10-24T21:35:25.228Z1000: a8:40:25:4c:82:ea
7492023-10-24T21:35:25.260Z1: a8:40:25:4c:82:eb
7502023-10-24T21:35:25.290Zproxy arp:
7512023-10-24T21:35:25.315Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
7522023-10-24T21:35:25.339Z+ retry=0
7532023-10-24T21:35:25.364Z+++ zoneadm list -n
7542023-10-24T21:35:25.388Z+++ grep oxz_ntp
7552023-10-24T21:35:25.413Z++ pfexec svcs -z -Hostate oxide/ntp
7562023-10-24T21:35:25.438Zsvcs: zone '-Hostate': entity not found
7572023-10-24T21:35:25.462Z++ true
7582023-10-24T21:35:25.487Z+ [[ '' != online ]]
7592023-10-24T21:35:25.512Z+ [[ 0 -gt 60 ]]
7602023-10-24T21:35:25.552Z+ sleep 1
7612023-10-24T21:35:25.770Z+ retry=1
7622023-10-24T21:35:25.796Z+++ zoneadm list -n
7632023-10-24T21:35:25.820Z+++ grep oxz_ntp
7642023-10-24T21:35:25.845Z++ pfexec svcs -z -Hostate oxide/ntp
7652023-10-24T21:35:25.871Zsvcs: zone '-Hostate': entity not found
7662023-10-24T21:35:25.895Z++ true
7672023-10-24T21:35:25.920Z+ [[ '' != online ]]
7682023-10-24T21:35:25.946Z+ [[ 1 -gt 60 ]]
7692023-10-24T21:35:25.970Z+ sleep 1
7702023-10-24T21:35:26.791Z+ retry=2
7712023-10-24T21:35:26.816Z+++ zoneadm list -n
7722023-10-24T21:35:26.840Z+++ grep oxz_ntp
7732023-10-24T21:35:26.866Z++ pfexec svcs -z -Hostate oxide/ntp
7742023-10-24T21:35:26.890Zsvcs: zone '-Hostate': entity not found
7752023-10-24T21:35:26.914Z++ true
7762023-10-24T21:35:26.939Z+ [[ '' != online ]]
7772023-10-24T21:35:26.963Z+ [[ 2 -gt 60 ]]
7782023-10-24T21:35:26.992Z+ sleep 1
7792023-10-24T21:35:27.812Z+ retry=3
7802023-10-24T21:35:27.838Z+++ zoneadm list -n
7812023-10-24T21:35:27.862Z+++ grep oxz_ntp
7822023-10-24T21:35:27.887Z++ pfexec svcs -z -Hostate oxide/ntp
7832023-10-24T21:35:27.914Zsvcs: zone '-Hostate': entity not found
7842023-10-24T21:35:27.939Z++ true
7852023-10-24T21:35:27.964Z+ [[ '' != online ]]
7862023-10-24T21:35:27.988Z+ [[ 3 -gt 60 ]]
7872023-10-24T21:35:28.013Z+ sleep 1
7882023-10-24T21:35:28.836Z+ retry=4
7892023-10-24T21:35:28.864Z+++ zoneadm list -n
7902023-10-24T21:35:28.896Z+++ grep oxz_ntp
7912023-10-24T21:35:28.926Z++ pfexec svcs -z -Hostate oxide/ntp
7922023-10-24T21:35:28.956Zsvcs: zone '-Hostate': entity not found
7932023-10-24T21:35:28.982Z++ true
7942023-10-24T21:35:29.008Z+ [[ '' != online ]]
7952023-10-24T21:35:29.033Z+ [[ 4 -gt 60 ]]
7962023-10-24T21:35:29.057Z+ sleep 1
7972023-10-24T21:35:29.856Z+ retry=5
7982023-10-24T21:35:29.882Z+++ zoneadm list -n
7992023-10-24T21:35:29.906Z+++ grep oxz_ntp
8002023-10-24T21:35:29.931Z++ pfexec svcs -z -Hostate oxide/ntp
8012023-10-24T21:35:29.956Zsvcs: zone '-Hostate': entity not found
8022023-10-24T21:35:29.980Z++ true
8032023-10-24T21:35:30.008Z+ [[ '' != online ]]
8042023-10-24T21:35:30.033Z+ [[ 5 -gt 60 ]]
8052023-10-24T21:35:30.059Z+ sleep 1
8062023-10-24T21:35:30.876Z+ retry=6
8072023-10-24T21:35:30.902Z+++ zoneadm list -n
8082023-10-24T21:35:30.946Z+++ grep oxz_ntp
8092023-10-24T21:35:30.972Z++ pfexec svcs -z -Hostate oxide/ntp
8102023-10-24T21:35:30.997Zsvcs: zone '-Hostate': entity not found
8112023-10-24T21:35:31.022Z++ true
8122023-10-24T21:35:31.047Z+ [[ '' != online ]]
8132023-10-24T21:35:31.071Z+ [[ 6 -gt 60 ]]
8142023-10-24T21:35:31.095Z+ sleep 1
8152023-10-24T21:35:31.899Z+ retry=7
8162023-10-24T21:35:31.935Z+++ zoneadm list -n
8172023-10-24T21:35:31.960Z+++ grep oxz_ntp
8182023-10-24T21:35:31.985Z++ pfexec svcs -z -Hostate oxide/ntp
8192023-10-24T21:35:32.010Zsvcs: zone '-Hostate': entity not found
8202023-10-24T21:35:32.034Z++ true
8212023-10-24T21:35:32.058Z+ [[ '' != online ]]
8222023-10-24T21:35:32.083Z+ [[ 7 -gt 60 ]]
8232023-10-24T21:35:32.107Z+ sleep 1
8242023-10-24T21:35:32.920Z+ retry=8
8252023-10-24T21:35:32.945Z+++ zoneadm list -n
8262023-10-24T21:35:32.974Z+++ grep oxz_ntp
8272023-10-24T21:35:33.000Z++ pfexec svcs -z -Hostate oxide/ntp
8282023-10-24T21:35:33.029Zsvcs: zone '-Hostate': entity not found
8292023-10-24T21:35:33.065Z++ true
8302023-10-24T21:35:33.092Z+ [[ '' != online ]]
8312023-10-24T21:35:33.132Z+ [[ 8 -gt 60 ]]
8322023-10-24T21:35:33.160Z+ sleep 1
8332023-10-24T21:35:33.941Z+ retry=9
8342023-10-24T21:35:33.965Z+++ zoneadm list -n
8352023-10-24T21:35:33.990Z+++ grep oxz_ntp
8362023-10-24T21:35:34.015Z++ pfexec svcs -z -Hostate oxide/ntp
8372023-10-24T21:35:34.039Zsvcs: zone '-Hostate': entity not found
8382023-10-24T21:35:34.064Z++ true
8392023-10-24T21:35:34.089Z+ [[ '' != online ]]
8402023-10-24T21:35:34.115Z+ [[ 9 -gt 60 ]]
8412023-10-24T21:35:34.152Z+ sleep 1
8422023-10-24T21:35:34.962Z+ retry=10
8432023-10-24T21:35:34.987Z+++ zoneadm list -n
8442023-10-24T21:35:35.012Z+++ grep oxz_ntp
8452023-10-24T21:35:35.036Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8462023-10-24T21:35:35.061Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
8472023-10-24T21:35:35.085Z++ true
8482023-10-24T21:35:35.114Z+ [[ '' != online ]]
8492023-10-24T21:35:35.138Z+ [[ 10 -gt 60 ]]
8502023-10-24T21:35:35.163Z+ sleep 1
8512023-10-24T21:35:35.995Z+ retry=11
8522023-10-24T21:35:36.020Z+++ zoneadm list -n
8532023-10-24T21:35:36.045Z+++ grep oxz_ntp
8542023-10-24T21:35:36.070Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8552023-10-24T21:35:36.094Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
8562023-10-24T21:35:36.120Z++ true
8572023-10-24T21:35:36.145Z+ [[ '' != online ]]
8582023-10-24T21:35:36.169Z+ [[ 11 -gt 60 ]]
8592023-10-24T21:35:36.196Z+ sleep 1
8602023-10-24T21:35:37.025Z+ retry=12
8612023-10-24T21:35:37.050Z+++ zoneadm list -n
8622023-10-24T21:35:37.074Z+++ grep oxz_ntp
8632023-10-24T21:35:37.099Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8642023-10-24T21:35:37.127Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
8652023-10-24T21:35:37.157Z++ true
8662023-10-24T21:35:37.186Z+ [[ '' != online ]]
8672023-10-24T21:35:37.210Z+ [[ 12 -gt 60 ]]
8682023-10-24T21:35:37.235Z+ sleep 1
8692023-10-24T21:35:38.054Z+ retry=13
8702023-10-24T21:35:38.079Z+++ zoneadm list -n
8712023-10-24T21:35:38.103Z+++ grep oxz_ntp
8722023-10-24T21:35:38.128Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8732023-10-24T21:35:38.153Z+ [[ disabled != online ]]
8742023-10-24T21:35:38.180Z+ [[ 13 -gt 60 ]]
8752023-10-24T21:35:38.204Z+ sleep 1
8762023-10-24T21:35:39.077Z+ retry=14
8772023-10-24T21:35:39.101Z+++ zoneadm list -n
8782023-10-24T21:35:39.126Z+++ grep oxz_ntp
8792023-10-24T21:35:39.150Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8802023-10-24T21:35:39.175Z+ [[ disabled != online ]]
8812023-10-24T21:35:39.200Z+ [[ 14 -gt 60 ]]
8822023-10-24T21:35:39.225Z+ sleep 1
8832023-10-24T21:35:40.103Z+ retry=15
8842023-10-24T21:35:40.135Z+++ zoneadm list -n
8852023-10-24T21:35:40.166Z+++ grep oxz_ntp
8862023-10-24T21:35:40.199Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8872023-10-24T21:35:40.236Z+ [[ disabled != online ]]
8882023-10-24T21:35:40.267Z+ [[ 15 -gt 60 ]]
8892023-10-24T21:35:40.297Z+ sleep 1
8902023-10-24T21:35:41.124Z+ retry=16
8912023-10-24T21:35:41.155Z+++ zoneadm list -n
8922023-10-24T21:35:41.182Z+++ grep oxz_ntp
8932023-10-24T21:35:41.206Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
8942023-10-24T21:35:41.231Z+ [[ disabled != online ]]
8952023-10-24T21:35:41.258Z+ [[ 16 -gt 60 ]]
8962023-10-24T21:35:41.285Z+ sleep 1
8972023-10-24T21:35:42.143Z+ retry=17
8982023-10-24T21:35:42.168Z+++ zoneadm list -n
8992023-10-24T21:35:42.193Z+++ grep oxz_ntp
9002023-10-24T21:35:42.220Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
9012023-10-24T21:35:42.247Z+ [[ disabled != online ]]
9022023-10-24T21:35:42.274Z+ [[ 17 -gt 60 ]]
9032023-10-24T21:35:42.299Z+ sleep 1
9042023-10-24T21:35:43.174Z+ retry=18
9052023-10-24T21:35:43.210Z+++ zoneadm list -n
9062023-10-24T21:35:43.242Z+++ grep oxz_ntp
9072023-10-24T21:35:43.269Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
9082023-10-24T21:35:43.295Z+ [[ disabled != online ]]
9092023-10-24T21:35:43.321Z+ [[ 18 -gt 60 ]]
9102023-10-24T21:35:43.346Z+ sleep 1
9112023-10-24T21:35:44.190Z+ retry=19
9122023-10-24T21:35:44.214Z+++ zoneadm list -n
9132023-10-24T21:35:44.242Z+++ grep oxz_ntp
9142023-10-24T21:35:44.267Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
9152023-10-24T21:35:44.291Z+ [[ disabled != online ]]
9162023-10-24T21:35:44.317Z+ [[ 19 -gt 60 ]]
9172023-10-24T21:35:44.341Z+ sleep 1
9182023-10-24T21:35:45.214Z+ retry=20
9192023-10-24T21:35:45.239Z+++ zoneadm list -n
9202023-10-24T21:35:45.275Z+++ grep oxz_ntp
9212023-10-24T21:35:45.300Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
9222023-10-24T21:35:45.324Z+ [[ disabled != online ]]
9232023-10-24T21:35:45.349Z+ [[ 20 -gt 60 ]]
9242023-10-24T21:35:45.374Z+ sleep 1
9252023-10-24T21:35:46.245Z+ retry=21
9262023-10-24T21:35:46.270Z+++ zoneadm list -n
9272023-10-24T21:35:46.294Z+++ grep oxz_ntp
9282023-10-24T21:35:46.319Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
9292023-10-24T21:35:46.344Z+ [[ disabled != online ]]
9302023-10-24T21:35:46.369Z+ [[ 21 -gt 60 ]]
9312023-10-24T21:35:46.401Z+ sleep 1
 
9322023-10-24T21:35:46.893Zsock console login: Oct 24 21:35:45 sok ip: ip: joining multicasts failed (4) on vopte0 - will use link layer broadcasts for multicast
 
9332023-10-24T21:35:47.269Z+ retry=22
9342023-10-24T21:35:47.293Z+++ zoneadm list -n
9352023-10-24T21:35:47.318Z+++ grep oxz_ntp
9362023-10-24T21:35:47.342Z++ pfexec svcs -z oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2 -Hostate oxide/ntp
9372023-10-24T21:35:47.367Z+ [[ online != online ]]
9382023-10-24T21:35:47.391Z+ echo 'Waited for chrony: 22s'
9392023-10-24T21:35:47.416ZWaited for chrony: 22s
9402023-10-24T21:35:47.440Z+ export RUST_BACKTRACE=1
9412023-10-24T21:35:47.465Z+ RUST_BACKTRACE=1
9422023-10-24T21:35:47.494Z+ export E2E_TLS_CERT IPPOOL_START IPPOOL_END
9432023-10-24T21:35:47.519Z+ ./tests/bootstrap
9442023-10-24T21:35:47.545ZUsing external DNS server at 10.151.2.164:53
9452023-10-24T21:35:47.573Z2023-10-24 21:35:46.435031040 UTC: attempting to log into API
9462023-10-24T21:36:02.345Z2023-10-24 21:36:02.425768115 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9472023-10-24T21:36:03.347Z2023-10-24 21:36:03.426644990 UTC: attempting to log into API
9482023-10-24T21:36:18.379Z2023-10-24 21:36:18.459903323 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9492023-10-24T21:36:19.379Z2023-10-24 21:36:19.460640285 UTC: attempting to log into API
9502023-10-24T21:36:34.414Z2023-10-24 21:36:34.458957543 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9512023-10-24T21:36:35.417Z2023-10-24 21:36:35.457988836 UTC: attempting to log into API
9522023-10-24T21:36:50.449Z2023-10-24 21:36:50.450001457 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9532023-10-24T21:36:51.449Z2023-10-24 21:36:51.448195642 UTC: attempting to log into API
9542023-10-24T21:37:06.487Z2023-10-24 21:37:06.475857026 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9552023-10-24T21:37:07.485Z2023-10-24 21:37:07.476620468 UTC: attempting to log into API
9562023-10-24T21:37:22.519Z2023-10-24 21:37:22.511017895 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9572023-10-24T21:37:23.520Z2023-10-24 21:37:23.511683346 UTC: attempting to log into API
9582023-10-24T21:37:38.553Z2023-10-24 21:37:38.543481296 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9592023-10-24T21:37:39.554Z2023-10-24 21:37:39.544400685 UTC: attempting to log into API
9602023-10-24T21:37:54.588Z2023-10-24 21:37:54.577593922 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9612023-10-24T21:37:55.589Z2023-10-24 21:37:55.578800564 UTC: attempting to log into API
9622023-10-24T21:38:10.622Z2023-10-24 21:38:10.611635911 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9632023-10-24T21:38:11.628Z2023-10-24 21:38:11.612701660 UTC: attempting to log into API
9642023-10-24T21:38:45.689Z2023-10-24 21:38:26.645972397 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9652023-10-24T21:38:46.101Z2023-10-24 21:38:27.647988036 UTC: attempting to log into API
9662023-10-24T21:38:46.235Z2023-10-24 21:38:42.692934222 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9672023-10-24T21:38:46.284Z2023-10-24 21:38:43.694090993 UTC: attempting to log into API
 
9682023-10-24T21:38:53.412ZOct 24 21:38:53 sock ip: ip: joining multicasts failed (4) on vopte3 - will use link layer broadcasts for multicast
 
9692023-10-24T21:38:58.738Z2023-10-24 21:38:58.727392329 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
9702023-10-24T21:38:59.739Z2023-10-24 21:38:59.728424177 UTC: attempting to log into API
 
9712023-10-24T21:39:01.703ZOct 24 21:39:01 sock ip: ip: joining multicasts failed (4) on vopte4 - will use link layer broadcasts for multicast
9722023-10-24T21:39:08.718ZOct 24 21:39:08 sok ip: ip: joining multicasts failed (4) on vopte5 - will use link layer broadcasts for multicast
 
9732023-10-24T21:39:14.772Z2023-10-24 21:39:14.761773214 UTC: login failed: logging in: error sending request for url (https://recovery.sys.oxide.test/v1/login/recovery/local): error trying to connect: operation timed out
 
9742023-10-24T21:39:15.451ZOct 24 21:39:15 sock ip: ip: joining multicasts failed (4) on vopte1 - will use link layer broadcasts for multicast
 
9752023-10-24T21:39:15.773Z2023-10-24 21:39:15.762965226 UTC: attempting to log into API
 
9762023-10-24T21:39:22.717ZOct 24 21:39:22 sock ip: ip: joining multicasts failed (4) on vopte2 - will use link layer broadcasts for multicast
 
9772023-10-24T21:39:26.560Z2023-10-24 21:39:26.550011748 UTC: login succeeded
9782023-10-24T21:39:26.592Zwaiting for nexus to come up...
9792023-10-24T21:39:26.657Zcreating IP pool... 10.151.2.175 - 10.151.2.227
9802023-10-24T21:39:26.709Zensuring datasets are ready...
9812023-10-24T21:39:29.359Zlet's roll.
9822023-10-24T21:39:29.851Z+ rm ./tests/bootstrap
9832023-10-24T21:39:29.875Z+ for test_bin in tests/*
9842023-10-24T21:39:29.899Z+ ./tests/bootstrap-49e31e75b930b622
9852023-10-24T21:39:29.923Z
9862023-10-24T21:39:29.948Zrunning 0 tests
9872023-10-24T21:39:29.978Z
9882023-10-24T21:39:30.011Ztest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s
9892023-10-24T21:39:30.036Z
9902023-10-24T21:39:30.071Z+ for test_bin in tests/*
9912023-10-24T21:39:30.097Z+ ./tests/end_to_end_tests-a91cf9af3457d280
9922023-10-24T21:39:30.122Z
9932023-10-24T21:39:30.148Zrunning 2 tests
9942023-10-24T21:39:30.173Ztest no_spoof::no_spoof_here ... ok
9952023-10-24T21:40:29.467Ztest instance_launch::instance_launch has been running for over 60 seconds
9962023-10-24T21:42:23.530Ztest instance_launch::instance_launch ... ok
9972023-10-24T21:42:23.555Z
9982023-10-24T21:42:23.580Ztest result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 174.06s
9992023-10-24T21:42:23.604Z
10002023-10-24T21:42:23.628Z+ _exit_trap
10012023-10-24T21:42:23.653Z+ local status=0
10022023-10-24T21:42:23.677Z+ [[ 0 -eq 0 ]]
10032023-10-24T21:42:23.701Z+ exit 0
10042023-10-24T21:42:23.725Zprocess exited: duration 484820 ms, exit code 0
 
10052023-10-24T21:42:23.805Zfound 47 output files
10062023-10-24T21:42:23.860Zuploading: /var/svc/log/oxide-sled-agent:default.log (459752 bytes)
10072023-10-24T21:42:24.569Zupload warning: file "/var/svc/log/oxide-sled-agent:default.log" changed size mid upload: 459752 -> 461002
10082023-10-24T21:42:25.641Zuploaded: /var/svc/log/oxide-sled-agent:default.log
10092023-10-24T21:42:25.665Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_af019375-954f-4133-a662-5183adb988e0/root/var/svc/log/oxide-cockroachdb:default.log (5369 bytes)
10102023-10-24T21:42:26.741Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_af019375-954f-4133-a662-5183adb988e0/root/var/svc/log/oxide-cockroachdb:default.log
10112023-10-24T21:42:26.765Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_080bc510-3bce-4c54-9ac8-8ca0e79a3018/root/var/svc/log/oxide-crucible-agent:default.log (14118 bytes)
10122023-10-24T21:42:27.880Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_080bc510-3bce-4c54-9ac8-8ca0e79a3018/root/var/svc/log/oxide-crucible-agent:default.log
10132023-10-24T21:42:27.904Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_080bc510-3bce-4c54-9ac8-8ca0e79a3018/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10142023-10-24T21:42:29.126Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_080bc510-3bce-4c54-9ac8-8ca0e79a3018/root/var/svc/log/oxide-crucible-downstairs:default.log
10152023-10-24T21:42:29.152Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_080bc510-3bce-4c54-9ac8-8ca0e79a3018/root/var/svc/log/oxide-crucible-downstairs:downstairs-e068d4fe-afd0-4bf4-bd65-d0c6c8429cef.log (5536 bytes)
10162023-10-24T21:42:30.069Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_080bc510-3bce-4c54-9ac8-8ca0e79a3018/root/var/svc/log/oxide-crucible-downstairs:downstairs-e068d4fe-afd0-4bf4-bd65-d0c6c8429cef.log
10172023-10-24T21:42:30.093Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_24eb9b59-ad29-4675-a63f-e750fd832712/root/var/svc/log/oxide-crucible-agent:default.log (3435 bytes)
10182023-10-24T21:42:31.164Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_24eb9b59-ad29-4675-a63f-e750fd832712/root/var/svc/log/oxide-crucible-agent:default.log
10192023-10-24T21:42:31.188Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_24eb9b59-ad29-4675-a63f-e750fd832712/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10202023-10-24T21:42:32.264Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_24eb9b59-ad29-4675-a63f-e750fd832712/root/var/svc/log/oxide-crucible-downstairs:default.log
10212023-10-24T21:42:32.290Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_6c535db9-c1ad-4403-9792-b57715a020e2/root/var/svc/log/oxide-crucible-agent:default.log (3436 bytes)
10222023-10-24T21:42:33.376Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_6c535db9-c1ad-4403-9792-b57715a020e2/root/var/svc/log/oxide-crucible-agent:default.log
10232023-10-24T21:42:33.402Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_6c535db9-c1ad-4403-9792-b57715a020e2/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10242023-10-24T21:42:34.478Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_6c535db9-c1ad-4403-9792-b57715a020e2/root/var/svc/log/oxide-crucible-downstairs:default.log
10252023-10-24T21:42:34.502Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_f5b3f475-73df-41ac-89e5-76edfb56834d/root/var/svc/log/oxide-crucible-agent:default.log (14118 bytes)
10262023-10-24T21:42:35.618Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_f5b3f475-73df-41ac-89e5-76edfb56834d/root/var/svc/log/oxide-crucible-agent:default.log
10272023-10-24T21:42:35.648Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_f5b3f475-73df-41ac-89e5-76edfb56834d/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10282023-10-24T21:42:36.720Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_f5b3f475-73df-41ac-89e5-76edfb56834d/root/var/svc/log/oxide-crucible-downstairs:default.log
10292023-10-24T21:42:36.745Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_f5b3f475-73df-41ac-89e5-76edfb56834d/root/var/svc/log/oxide-crucible-downstairs:downstairs-b66456ff-561e-4dc2-bc8e-985e8be400b0.log (5534 bytes)
10302023-10-24T21:42:37.838Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_f5b3f475-73df-41ac-89e5-76edfb56834d/root/var/svc/log/oxide-crucible-downstairs:downstairs-b66456ff-561e-4dc2-bc8e-985e8be400b0.log
10312023-10-24T21:42:37.862Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_c932fc0f-61b1-42b9-a0ab-d3b2134e6e17/root/var/svc/log/oxide-external_dns:default.log (15616 bytes)
10322023-10-24T21:42:39.003Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_c932fc0f-61b1-42b9-a0ab-d3b2134e6e17/root/var/svc/log/oxide-external_dns:default.log
10332023-10-24T21:42:39.030Zuploading: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_83d2958b-7990-4d4a-a0a8-4b5b30b5c3f5/root/var/svc/log/oxide-nexus:default.log (556318 bytes)
10342023-10-24T21:42:39.856Zupload warning: file "/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_83d2958b-7990-4d4a-a0a8-4b5b30b5c3f5/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 556318 -> 556926
10352023-10-24T21:42:40.928Zuploaded: /pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_83d2958b-7990-4d4a-a0a8-4b5b30b5c3f5/root/var/svc/log/oxide-nexus:default.log
10362023-10-24T21:42:40.956Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_aaf0b93c-bea7-47a1-9f91-a7fda9c6c0b8/root/var/svc/log/oxide-cockroachdb:default.log (5370 bytes)
10372023-10-24T21:42:42.022Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_aaf0b93c-bea7-47a1-9f91-a7fda9c6c0b8/root/var/svc/log/oxide-cockroachdb:default.log
10382023-10-24T21:42:42.046Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_9c679766-703b-4868-b75f-17d8e9624b4d/root/var/svc/log/oxide-crucible-agent:default.log (14113 bytes)
10392023-10-24T21:42:43.172Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_9c679766-703b-4868-b75f-17d8e9624b4d/root/var/svc/log/oxide-crucible-agent:default.log
10402023-10-24T21:42:43.199Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_9c679766-703b-4868-b75f-17d8e9624b4d/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10412023-10-24T21:42:44.268Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_9c679766-703b-4868-b75f-17d8e9624b4d/root/var/svc/log/oxide-crucible-downstairs:default.log
10422023-10-24T21:42:44.291Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_9c679766-703b-4868-b75f-17d8e9624b4d/root/var/svc/log/oxide-crucible-downstairs:downstairs-6dfb5f44-b0f8-4c31-bc41-68e31804e145.log (3051 bytes)
10432023-10-24T21:42:45.362Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_9c679766-703b-4868-b75f-17d8e9624b4d/root/var/svc/log/oxide-crucible-downstairs:downstairs-6dfb5f44-b0f8-4c31-bc41-68e31804e145.log
10442023-10-24T21:42:45.386Zuploading: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_e60639d0-241b-44f5-a3eb-3b1a96b2b7ea/root/var/svc/log/oxide-external_dns:default.log (15613 bytes)
10452023-10-24T21:42:46.523Zuploaded: /pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_external_dns_e60639d0-241b-44f5-a3eb-3b1a96b2b7ea/root/var/svc/log/oxide-external_dns:default.log
10462023-10-24T21:42:46.547Zuploading: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_pantry_3cd779bd-ff1e-4fe6-828e-123ef9b7d68a/root/var/svc/log/oxide-crucible-pantry:default.log (2338 bytes)
10472023-10-24T21:42:47.618Zuploaded: /pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone/oxz_crucible_pantry_3cd779bd-ff1e-4fe6-828e-123ef9b7d68a/root/var/svc/log/oxide-crucible-pantry:default.log
10482023-10-24T21:42:47.642Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_10613f58-2a6f-4b72-b261-e40cffb16769/root/var/svc/log/oxide-crucible-agent:default.log (3436 bytes)
10492023-10-24T21:42:48.735Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_10613f58-2a6f-4b72-b261-e40cffb16769/root/var/svc/log/oxide-crucible-agent:default.log
10502023-10-24T21:42:48.760Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_10613f58-2a6f-4b72-b261-e40cffb16769/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10512023-10-24T21:42:49.829Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_10613f58-2a6f-4b72-b261-e40cffb16769/root/var/svc/log/oxide-crucible-downstairs:default.log
10522023-10-24T21:42:49.853Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_b6f3d180-80bb-4812-879b-0ab601d99a11/root/var/svc/log/oxide-crucible-agent:default.log (14116 bytes)
10532023-10-24T21:42:50.947Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_b6f3d180-80bb-4812-879b-0ab601d99a11/root/var/svc/log/oxide-crucible-agent:default.log
10542023-10-24T21:42:50.971Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_b6f3d180-80bb-4812-879b-0ab601d99a11/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10552023-10-24T21:42:52.041Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_b6f3d180-80bb-4812-879b-0ab601d99a11/root/var/svc/log/oxide-crucible-downstairs:default.log
10562023-10-24T21:42:52.066Zuploading: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_b6f3d180-80bb-4812-879b-0ab601d99a11/root/var/svc/log/oxide-crucible-downstairs:downstairs-922ec25c-8ebe-45ed-85e7-778a0b51fb6c.log (3050 bytes)
10572023-10-24T21:42:53.138Zuploaded: /pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone/oxz_crucible_b6f3d180-80bb-4812-879b-0ab601d99a11/root/var/svc/log/oxide-crucible-downstairs:downstairs-922ec25c-8ebe-45ed-85e7-778a0b51fb6c.log
10582023-10-24T21:42:53.163Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_7a54bd7c-b4d9-45a6-8267-d9fdfb23bb54/root/var/svc/log/oxide-cockroachdb:default.log (5369 bytes)
10592023-10-24T21:42:54.235Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_7a54bd7c-b4d9-45a6-8267-d9fdfb23bb54/root/var/svc/log/oxide-cockroachdb:default.log
10602023-10-24T21:42:54.260Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_d49f5a89-9871-460e-ab42-25d8a5f46df2/root/var/svc/log/oxide-cockroachdb:default.log (5369 bytes)
10612023-10-24T21:42:55.340Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_cockroachdb_d49f5a89-9871-460e-ab42-25d8a5f46df2/root/var/svc/log/oxide-cockroachdb:default.log
10622023-10-24T21:42:55.364Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_773b69a7-5a46-4954-a7a4-4ac8a87bf2ad/root/var/svc/log/oxide-crucible-agent:default.log (3436 bytes)
10632023-10-24T21:42:56.436Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_773b69a7-5a46-4954-a7a4-4ac8a87bf2ad/root/var/svc/log/oxide-crucible-agent:default.log
10642023-10-24T21:42:56.465Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_773b69a7-5a46-4954-a7a4-4ac8a87bf2ad/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10652023-10-24T21:42:57.544Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_773b69a7-5a46-4954-a7a4-4ac8a87bf2ad/root/var/svc/log/oxide-crucible-downstairs:default.log
10662023-10-24T21:42:57.568Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_pantry_599765a0-a4fe-4f0b-9c76-f9c616c10dee/root/var/svc/log/oxide-crucible-pantry:default.log (2333 bytes)
10672023-10-24T21:42:58.637Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_crucible_pantry_599765a0-a4fe-4f0b-9c76-f9c616c10dee/root/var/svc/log/oxide-crucible-pantry:default.log
10682023-10-24T21:42:58.676Zuploading: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_nexus_341f5277-98d3-48ad-90d3-b446359dc31e/root/var/svc/log/oxide-nexus:default.log (3847515 bytes)
10692023-10-24T21:42:59.653Zupload warning: file "/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_nexus_341f5277-98d3-48ad-90d3-b446359dc31e/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 3847515 -> 3849340
10702023-10-24T21:43:00.724Zuploaded: /pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone/oxz_nexus_341f5277-98d3-48ad-90d3-b446359dc31e/root/var/svc/log/oxide-nexus:default.log
10712023-10-24T21:43:00.749Zuploading: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_internal_dns_3f3d5ef0-d196-4bca-9a01-7e9003a21c76/root/var/svc/log/oxide-internal_dns:default.log (99569 bytes)
10722023-10-24T21:43:01.932Zuploaded: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_internal_dns_3f3d5ef0-d196-4bca-9a01-7e9003a21c76/root/var/svc/log/oxide-internal_dns:default.log
10732023-10-24T21:43:01.961Zuploading: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_internal_dns_b8094cd1-f4ae-4460-8d2b-e8af4fbf6c3a/root/var/svc/log/oxide-internal_dns:default.log (105020 bytes)
10742023-10-24T21:43:03.162Zuploaded: /pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone/oxz_internal_dns_b8094cd1-f4ae-4460-8d2b-e8af4fbf6c3a/root/var/svc/log/oxide-internal_dns:default.log
10752023-10-24T21:43:03.187Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-agent:default.log (24827 bytes)
10762023-10-24T21:43:04.303Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-agent:default.log
10772023-10-24T21:43:04.328Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-downstairs:default.log (75 bytes)
10782023-10-24T21:43:05.397Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-downstairs:default.log
10792023-10-24T21:43:05.421Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-downstairs:downstairs-a31b8384-73d5-4ee6-a028-4b3695518729.log (5537 bytes)
10802023-10-24T21:43:06.491Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-downstairs:downstairs-a31b8384-73d5-4ee6-a028-4b3695518729.log
10812023-10-24T21:43:06.520Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-downstairs:downstairs-acc2c95d-6f76-4ab0-bd08-972bd6b57075.log (3051 bytes)
10822023-10-24T21:43:07.594Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_crucible_2ca5e7d9-e94e-40a9-bed9-395e71d76bbc/root/var/svc/log/oxide-crucible-downstairs:downstairs-acc2c95d-6f76-4ab0-bd08-972bd6b57075.log
10832023-10-24T21:43:07.621Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/svc/log/oxide-ntp:default.log (1443 bytes)
10842023-10-24T21:43:08.720Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/svc/log/oxide-ntp:default.log
10852023-10-24T21:43:08.745Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_oximeter_03c179f4-80e3-4ea8-abe6-5024445a2243/root/var/svc/log/oxide-oximeter:default.log (179869 bytes)
10862023-10-24T21:43:08.905Zupload warning: file "/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_oximeter_03c179f4-80e3-4ea8-abe6-5024445a2243/root/var/svc/log/oxide-oximeter:default.log" changed size mid upload: 179869 -> 216308
10872023-10-24T21:43:09.977Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_oximeter_03c179f4-80e3-4ea8-abe6-5024445a2243/root/var/svc/log/oxide-oximeter:default.log
10882023-10-24T21:43:10.003Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_2c1bcfe0-b238-4c08-94b4-8eb717ef446c/root/var/svc/log/oxide-cockroachdb:default.log (5343 bytes)
10892023-10-24T21:43:11.075Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_cockroachdb_2c1bcfe0-b238-4c08-94b4-8eb717ef446c/root/var/svc/log/oxide-cockroachdb:default.log
10902023-10-24T21:43:11.099Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_a7656976-4220-4287-bf0b-7480a96fe76f/root/var/svc/log/oxide-internal_dns:default.log (103946 bytes)
10912023-10-24T21:43:11.234Zupload warning: file "/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_a7656976-4220-4287-bf0b-7480a96fe76f/root/var/svc/log/oxide-internal_dns:default.log" changed size mid upload: 103946 -> 105198
10922023-10-24T21:43:12.304Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_internal_dns_a7656976-4220-4287-bf0b-7480a96fe76f/root/var/svc/log/oxide-internal_dns:default.log
10932023-10-24T21:43:12.355Zuploading: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_81a5bafd-cbf8-425d-8190-7720a01b2667/root/var/svc/log/oxide-nexus:default.log (857525 bytes)
10942023-10-24T21:43:12.590Zupload warning: file "/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_81a5bafd-cbf8-425d-8190-7720a01b2667/root/var/svc/log/oxide-nexus:default.log" changed size mid upload: 857525 -> 918748
10952023-10-24T21:43:13.662Zuploaded: /pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_nexus_81a5bafd-cbf8-425d-8190-7720a01b2667/root/var/svc/log/oxide-nexus:default.log
10962023-10-24T21:43:13.712Zuploading: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_849234da-c9c2-454e-9194-54f6131d1ec1/root/var/svc/log/oxide-clickhouse:default.log (708925 bytes)
10972023-10-24T21:43:13.965Zupload warning: file "/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_849234da-c9c2-454e-9194-54f6131d1ec1/root/var/svc/log/oxide-clickhouse:default.log" changed size mid upload: 708925 -> 871785
10982023-10-24T21:43:15.038Zuploaded: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_clickhouse_849234da-c9c2-454e-9194-54f6131d1ec1/root/var/svc/log/oxide-clickhouse:default.log
10992023-10-24T21:43:15.062Zuploading: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_pantry_23f3b627-21eb-4e89-9f73-d8adca41dafe/root/var/svc/log/oxide-crucible-pantry:default.log (2337 bytes)
11002023-10-24T21:43:16.135Zuploaded: /pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone/oxz_crucible_pantry_23f3b627-21eb-4e89-9f73-d8adca41dafe/root/var/svc/log/oxide-crucible-pantry:default.log
11012023-10-24T21:43:16.159Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/measurements.log (20002 bytes)
11022023-10-24T21:43:16.228Zupload warning: file "/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/measurements.log" changed size mid upload: 20002 -> 21098
11032023-10-24T21:43:17.300Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/measurements.log
11042023-10-24T21:43:17.324Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/statistics.log (12987 bytes)
11052023-10-24T21:43:17.371Zupload warning: file "/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/statistics.log" changed size mid upload: 12987 -> 13923
11062023-10-24T21:43:18.452Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/statistics.log
11072023-10-24T21:43:18.479Zuploading: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/tracking.log (2119 bytes)
11082023-10-24T21:43:19.565Zuploaded: /pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone/oxz_ntp_6bd0f613-e6db-4c69-92e4-39acc4003da2/root/var/log/chrony/tracking.log