01KVP0W5G9M5NX5JAHMEBWYNHZ: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-3.0-opte-0.41",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KVP0YEZZ5VXYED8GBTDYDGFH

Tags:

Output:

SEQ GLOBAL TIME DETAILS
12026-06-21T21:40:58.806Zjob dependencies complete; ready to run (waiting for 19 m 36 s)
22026-06-22T19:08:10.013Zjob assigned to worker 01KVRBNAC17QF4MB4ANBGRH6K0 [factory lab, sock/17309] (queued for 21 h 27 m 11 s)
32026-06-22T19:08:10.046Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-06-22T19:08:10.084Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52026-06-22T19:08:10.118Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62026-06-22T19:08:10.173Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72026-06-22T19:08:10.207Zhttp://172.20.0.5:9969/postboot/sock... ok
82026-06-22T19:08:10.240ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92026-06-22T19:08:10.276ZOxide Helios Version helios-3.0.24018 64-bit
102026-06-22T19:08:10.307ZConfiguring devices.
112026-06-22T19:08:10.340Z##################################################
122026-06-22T19:08:10.374Z#### ############# ##
132026-06-22T19:08:10.407Z### ### ############ ##
142026-06-22T19:08:10.441Z## ### # ## ### ## Oxide Computer Company ##
152026-06-22T19:08:10.474Z## ## ## ### # ### ##
162026-06-22T19:08:10.508Z## # ### #### #### This Station Under ##
172026-06-22T19:08:10.541Z### ### #### # ### Computer Control ##
182026-06-22T19:08:10.575Z#### #### ### ## ##
192026-06-22T19:08:10.608Z##################################################
202026-06-22T19:08:10.641ZHostname: sock
212026-06-22T19:08:10.783ZLoading smf(7) service descriptions:
222026-06-22T19:08:11.458Zdownloading input: /input/package/work/package.tar.gz
232026-06-22T19:15:16.668Zdownloaded input: /input/package/work/package.tar.gz
 
242026-06-22T19:15:16.692Zstarting task 0: "setup"
252026-06-22T19:15:16.692Z++ uname -s
262026-06-22T19:15:16.692Z+ kern=SunOS
272026-06-22T19:15:16.692Z+ build_user=build
282026-06-22T19:15:16.692Z+ build_uid=12345
292026-06-22T19:15:16.693Z+ work_dir=/work
302026-06-22T19:15:16.693Z+ input_dir=/input
312026-06-22T19:15:16.693Z+ [[ 0 == 12345 ]]
322026-06-22T19:15:16.693Z+ case "$kern" in
332026-06-22T19:15:16.693Z+ groupadd -g 12345 build
342026-06-22T19:15:16.693Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
352026-06-22T19:15:18.687Z+ zfs create -o mountpoint=/work rpool/work
362026-06-22T19:15:18.735Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
372026-06-22T19:15:18.759Z+ home_fs=zfs
382026-06-22T19:15:18.759Z+ [[ zfs == autofs ]]
392026-06-22T19:15:18.759Z+ mkdir -p /home/build
402026-06-22T19:15:18.759Z+ chown build:build /home/build /work
412026-06-22T19:15:20.743Z+ chmod 0700 /home/build /work
422026-06-22T19:15:20.767Zprocess exited: duration 4079 ms, exit code 0
 
432026-06-22T19:15:20.815Zstarting task 1: "authentication"
442026-06-22T19:15:20.864Zprocess exited: duration 48 ms, exit code 0
 
452026-06-22T19:15:20.912Zstarting task 2: "build"
462026-06-22T19:15:20.936Z+ trap _exit_trap EXIT
472026-06-22T19:15:20.937Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
482026-06-22T19:15:21.032Zsecpolicy_fs_linkdir: 0x55 = 0x55
492026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
502026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
512026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+3: 0 = 0xe5
522026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+4: 0 = 0xb8
532026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+5: 0 = 0x1
542026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
552026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
562026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
572026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
582026-06-22T19:15:21.032Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
592026-06-22T19:15:21.057Z+ [[ -d /opt/oxide ]]
602026-06-22T19:15:21.057Z+ pfexec mv /opt/oxide /opt/oxide-underneath
612026-06-22T19:15:21.057Z+ pfexec mkdir /opt/oxide
622026-06-22T19:15:21.057Z+ pfexec mount -F tmpfs -O swap /opt/oxide
632026-06-22T19:15:21.057Z+ [[ -d /opt/oxide-underneath ]]
642026-06-22T19:15:21.057Z+ cd /opt/oxide-underneath
652026-06-22T19:15:21.057Z+ pfexec tar ceEp@/f - .
662026-06-22T19:15:21.057Z+ cd /opt/oxide
672026-06-22T19:15:21.057Z+ pfexec tar xveEp@/f -
682026-06-22T19:15:21.107Zx ., 0 bytes, 0 tape blocks
692026-06-22T19:15:21.107Zx ./opte, 0 bytes, 0 tape blocks
702026-06-22T19:15:21.107Zx ./opte/bin, 0 bytes, 0 tape blocks
712026-06-22T19:15:21.108Zx ./opte/bin/opteadm, 24740168 bytes, 48321 tape blocks
722026-06-22T19:15:21.131Z+ pfexec rm -rf /opt/oxide-underneath
732026-06-22T19:15:21.131Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
742026-06-22T19:15:21.155Z+ pfexec mkdir /opt/oxide/work
752026-06-22T19:15:21.155Z+ pfexec chown build:build /opt/oxide/work
762026-06-22T19:15:21.155Z+ cd /opt/oxide/work
772026-06-22T19:15:21.155Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
782026-06-22T19:15:35.988ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
792026-06-22T19:15:35.988Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
802026-06-22T19:15:35.988Zx out/target/test, 107 bytes, 1 tape blocks
812026-06-22T19:15:35.988Zx out/npuzone/npuzone, 28059200 bytes, 54804 tape blocks
822026-06-22T19:15:35.988Zx package-manifest.toml, 35397 bytes, 70 tape blocks
832026-06-22T19:15:35.988Zx smf/sled-agent/non-gimlet/config.toml, 5355 bytes, 11 tape blocks
842026-06-22T19:15:35.988Zx target/release/omicron-package, 46321528 bytes, 90472 tape blocks
852026-06-22T19:15:35.988Zx target/release/xtask, 11727472 bytes, 22906 tape blocks
862026-06-22T19:15:35.989Zx target/debug/bootstrap, 364712912 bytes, 712330 tape blocks
872026-06-22T19:15:35.989Zx tests/bootstrap-924fc3fc47321669, 31963304 bytes, 62429 tape blocks
882026-06-22T19:15:35.989Zx tests/commtest-408c589a9be9a15a, 33074240 bytes, 64599 tape blocks
892026-06-22T19:15:35.989Zx tests/dhcp_server-f35f394bfb169ef7, 31963320 bytes, 62429 tape blocks
902026-06-22T19:15:35.989Zx tests/end_to_end_tests-f00a927a2ca939a9, 407545408 bytes, 795988 tape blocks
912026-06-22T19:15:35.989Zx tools/opte_version, 9 bytes, 1 tape blocks
922026-06-22T19:15:35.989Zx tools/opte_version_override, 1245 bytes, 3 tape blocks
932026-06-22T19:15:35.989Zx out/clickhouse.tar.gz, 321027423 bytes, 627007 tape blocks
942026-06-22T19:15:35.989Zx out/clickhouse_keeper.tar.gz, 305726949 bytes, 597123 tape blocks
952026-06-22T19:15:35.989Zx out/clickhouse_server.tar.gz, 321024816 bytes, 627002 tape blocks
962026-06-22T19:15:35.989Zx out/cockroachdb.tar.gz, 161497075 bytes, 315424 tape blocks
972026-06-22T19:15:35.989Zx out/crucible-pantry-zone.tar.gz, 37358475 bytes, 72966 tape blocks
982026-06-22T19:15:35.989Zx out/crucible-utils.tar, 16872960 bytes, 32955 tape blocks
992026-06-22T19:15:35.989Zx out/crucible-zone.tar.gz, 53302102 bytes, 104106 tape blocks
1002026-06-22T19:15:35.989Zx out/external-dns.tar.gz, 43945929 bytes, 85832 tape blocks
1012026-06-22T19:15:35.989Zx out/internal-dns.tar.gz, 43945525 bytes, 85832 tape blocks
1022026-06-22T19:15:35.989Zx out/mg-ddm-gz.tar, 100266496 bytes, 195833 tape blocks
1032026-06-22T19:15:35.989Zx out/nexus.tar.gz, 283655207 bytes, 554015 tape blocks
1042026-06-22T19:15:35.989Zx out/ntp.tar.gz, 24702033 bytes, 48247 tape blocks
1052026-06-22T19:15:35.990Zx out/omicron-gateway-softnpu.tar.gz, 30938786 bytes, 60428 tape blocks
1062026-06-22T19:15:35.990Zx out/omicron-sled-agent.tar, 199506944 bytes, 389662 tape blocks
1072026-06-22T19:15:35.990Zx out/overlay.tar.gz, 3030 bytes, 6 tape blocks
1082026-06-22T19:15:35.990Zx out/oximeter.tar.gz, 63811021 bytes, 124631 tape blocks
1092026-06-22T19:15:35.990Zx out/oxlog.tar, 3872768 bytes, 7564 tape blocks
1102026-06-22T19:15:35.990Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1112026-06-22T19:15:35.990Zx out/propolis-server.tar.gz, 35755910 bytes, 69836 tape blocks
1122026-06-22T19:15:35.990Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1132026-06-22T19:15:35.990Zx out/switch-softnpu.tar.gz, 458026174 bytes, 894583 tape blocks
1142026-06-22T19:15:36.015Z
1152026-06-22T19:15:36.015Zreal 14.823459928
1162026-06-22T19:15:36.015Zuser 12.422575993
1172026-06-22T19:15:36.015Zsys 4.184446215
1182026-06-22T19:15:36.015Ztrap 0.000701151
1192026-06-22T19:15:36.015Ztflt 0.000000000
1202026-06-22T19:15:36.015Zdflt 0.000000000
1212026-06-22T19:15:36.015Zkflt 0.000000000
1222026-06-22T19:15:36.015Zlock 0.000000000
1232026-06-22T19:15:36.015Zslp 12.418511127
1242026-06-22T19:15:36.015Zlat 0.618522543
1252026-06-22T19:15:36.015Zstop 0.000134231
1262026-06-22T19:15:36.015Z+ source .github/buildomat/ci-env.sh
1272026-06-22T19:15:36.015Z++ export CARGO_TERM_COLOR=always
1282026-06-22T19:15:36.015Z++ CARGO_TERM_COLOR=always
1292026-06-22T19:15:36.015Z+ source tools/opte_version_override
1302026-06-22T19:15:36.015Z++ OPTE_COMMIT=
1312026-06-22T19:15:36.015Z+ [[ x != \x ]]
1322026-06-22T19:15:36.015Z+ bmat address ls
1332026-06-22T19:15:36.015ZNAME CIDR FIRST LAST COUNT
1342026-06-22T19:15:36.015Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1352026-06-22T19:15:36.015Z++ bmat address ls -f extra -Ho first,last,count
1362026-06-22T19:15:36.015Z+ set -- 10.151.2.164 10.151.2.227 64
1372026-06-22T19:15:36.015Z+ EXTRA_IP_START=10.151.2.164
1382026-06-22T19:15:36.015Z+ EXTRA_IP_END=10.151.2.227
1392026-06-22T19:15:36.015Z+ EXTRA_IP_COUNT=64
1402026-06-22T19:15:36.015Z+ (( EXTRA_IP_COUNT >= 32 ))
1412026-06-22T19:15:36.015Z+ EXTRA_IP_BASE=10.151.2
1422026-06-22T19:15:36.015Z+ EXTRA_IP_FOCTET=164
1432026-06-22T19:15:36.017Z+ EXTRA_IP_LOCTET=227
1442026-06-22T19:15:36.017Z+ SERVICE_IP_POOL_START=10.151.2.164
1452026-06-22T19:15:36.017Z+ SERVICE_IP_POOL_END=10.151.2.173
1462026-06-22T19:15:36.017Z+ DNS_IP1=10.151.2.164
1472026-06-22T19:15:36.017Z+ DNS_IP2=10.151.2.165
1482026-06-22T19:15:36.017Z+ UPLINK_IP=10.151.2.174
1492026-06-22T19:15:36.017Z+ IPPOOL_START=10.151.2.175
1502026-06-22T19:15:36.017Z+ IPPOOL_END=10.151.2.227
1512026-06-22T19:15:36.017Z++ ipadm show-addr -po type,addr
1522026-06-22T19:15:36.017Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1532026-06-22T19:15:36.017Z+ GATEWAY_IP=10.151.2.100
1542026-06-22T19:15:36.017Z+ [[ -n 10.151.2.100 ]]
1552026-06-22T19:15:36.017Z+ ping -s 10.151.2.100 56 1
1562026-06-22T19:15:36.017ZPING 10.151.2.100: 56 data bytes
1572026-06-22T19:15:36.041Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.013 ms
1582026-06-22T19:15:36.041Z
1592026-06-22T19:15:36.041Z----10.151.2.100 PING Statistics----
1602026-06-22T19:15:36.041Z1 packets transmitted, 1 packets received, 0% packet loss
1612026-06-22T19:15:36.041Zround-trip (ms) min/avg/max/stddev = 0.013/0.013/0.013/-nan
1622026-06-22T19:15:36.041Z++ arp -an
1632026-06-22T19:15:36.041Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1642026-06-22T19:15:36.041Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1652026-06-22T19:15:36.042Z+ routeadm -e ipv4-forwarding -u
1662026-06-22T19:15:36.130Z+ PXA_START=10.151.2.164
1672026-06-22T19:15:36.130Z+ PXA_END=10.151.2.227
1682026-06-22T19:15:36.130Z+ DISKS=($(pfexec nvmeadm list -p -o disk))
1692026-06-22T19:15:36.154Z++ pfexec nvmeadm list -p -o disk
1702026-06-22T19:15:36.154Z+ pfexec zpool create -f scratch c1t0026B728281787A0d0 c2t0026B72828178F30d0
1712026-06-22T19:15:36.317Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1722026-06-22T19:15:36.342ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1732026-06-22T19:15:36.390Zcreating virtual hardware
1742026-06-22T19:15:36.414Zcreating /scratch/m2_0.vdev
1752026-06-22T19:15:36.414Zcreating /scratch/m2_1.vdev
1762026-06-22T19:15:36.414Zcreating /scratch/u2_0.vdev
1772026-06-22T19:15:36.414Zcreating /scratch/u2_1.vdev
1782026-06-22T19:15:36.414Zcreating /scratch/u2_2.vdev
1792026-06-22T19:15:36.414Zcreating /scratch/u2_3.vdev
1802026-06-22T19:15:36.415Zcreating /scratch/u2_4.vdev
1812026-06-22T19:15:36.415Zcreating /scratch/u2_5.vdev
1822026-06-22T19:15:36.415Zcreating /scratch/u2_6.vdev
1832026-06-22T19:15:36.415Zcreating /scratch/u2_7.vdev
1842026-06-22T19:15:36.415Zcreating /scratch/u2_8.vdev
1852026-06-22T19:15:36.439ZSimnet net0/sc0_0 exists
1862026-06-22T19:15:36.462ZSimnet net1/sc1_0 exists
1872026-06-22T19:15:36.462ZVnic sc0_1 exists
1882026-06-22T19:15:51.635ZUsing 10.151.2.100 as gateway ip
1892026-06-22T19:15:51.635Zusing 18:C0:4D:0D:A0:2A as gateway mac
1902026-06-22T19:15:51.635Zconfiguring SoftNPU ARP entry
1912026-06-22T19:15:51.662Zconfiguring SoftNPU proxy ARP
1922026-06-22T19:15:51.716ZSoftNPU state:
1932026-06-22T19:15:51.716Z local v6:
1942026-06-22T19:15:51.716Z local v4:
1952026-06-22T19:15:51.716Z router v6_idx:
1962026-06-22T19:15:51.716Z router v6_routes:
1972026-06-22T19:15:51.716Z router v4_idx:
1982026-06-22T19:15:51.716Z router v4_routes:
1992026-06-22T19:15:51.716Z resolver v4:
2002026-06-22T19:15:51.716Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
2012026-06-22T19:15:51.716Z resolver v6:
2022026-06-22T19:15:51.716Z nat v4:
2032026-06-22T19:15:51.716Z nat_v6:
2042026-06-22T19:15:51.716Z attached subnet v4:
2052026-06-22T19:15:51.716Z attached subnet v6:
2062026-06-22T19:15:51.716Z port_mac:
2072026-06-22T19:15:51.716Z proxy arp:
2082026-06-22T19:15:51.716Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2092026-06-22T19:15:51.716Zcreated virtual hardware
2102026-06-22T19:15:51.741Z
2112026-06-22T19:15:51.741Zreal 15.394211986
2122026-06-22T19:15:51.741Zuser 0.575118625
2132026-06-22T19:15:51.741Zsys 3.441463323
2142026-06-22T19:15:51.741Ztrap 0.000067868
2152026-06-22T19:15:51.741Ztflt 0.000205704
2162026-06-22T19:15:51.741Zdflt 0.000578257
2172026-06-22T19:15:51.741Zkflt 0.000012078
2182026-06-22T19:15:51.741Zlock 12:02.137333026
2192026-06-22T19:15:51.741Zslp 46.576124646
2202026-06-22T19:15:51.741Zlat 0.191974049
2212026-06-22T19:15:51.741Zstop 0.251506504
2222026-06-22T19:15:51.742Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2232026-06-22T19:15:51.766Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2242026-06-22T19:15:51.766Z+ diff -u pkg/config.toml~ pkg/config.toml
2252026-06-22T19:15:51.791Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2262026-06-22T19:15:51.791Z+++ pkg/config.toml Mon Jun 22 19:15:51 2026
2272026-06-22T19:15:51.791Z@@ -29,18 +29,18 @@
2282026-06-22T19:15:51.791Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2292026-06-22T19:15:51.791Z # followed by a string that is embedded into their fake serial values.
2302026-06-22T19:15:51.791Z vdevs = [
2312026-06-22T19:15:51.791Z- "m2_0.vdev",
2322026-06-22T19:15:51.791Z- "m2_1.vdev",
2332026-06-22T19:15:51.791Z+ "/scratch/m2_0.vdev",
2342026-06-22T19:15:51.791Z+ "/scratch/m2_1.vdev",
2352026-06-22T19:15:51.791Z
2362026-06-22T19:15:51.791Z- "u2_0.vdev",
2372026-06-22T19:15:51.791Z- "u2_1.vdev",
2382026-06-22T19:15:51.791Z- "u2_2.vdev",
2392026-06-22T19:15:51.791Z- "u2_3.vdev",
2402026-06-22T19:15:51.791Z- "u2_4.vdev",
2412026-06-22T19:15:51.791Z- "u2_5.vdev",
2422026-06-22T19:15:51.792Z- "u2_6.vdev",
2432026-06-22T19:15:51.792Z- "u2_7.vdev",
2442026-06-22T19:15:51.792Z- "u2_8.vdev",
2452026-06-22T19:15:51.792Z+ "/scratch/u2_0.vdev",
2462026-06-22T19:15:51.792Z+ "/scratch/u2_1.vdev",
2472026-06-22T19:15:51.792Z+ "/scratch/u2_2.vdev",
2482026-06-22T19:15:51.792Z+ "/scratch/u2_3.vdev",
2492026-06-22T19:15:51.792Z+ "/scratch/u2_4.vdev",
2502026-06-22T19:15:51.792Z+ "/scratch/u2_5.vdev",
2512026-06-22T19:15:51.792Z+ "/scratch/u2_6.vdev",
2522026-06-22T19:15:51.792Z+ "/scratch/u2_7.vdev",
2532026-06-22T19:15:51.792Z+ "/scratch/u2_8.vdev",
2542026-06-22T19:15:51.792Z ]
2552026-06-22T19:15:51.792Z
2562026-06-22T19:15:51.792Z # The amount of memory held back for services which exist between zero and one
2572026-06-22T19:15:51.792Z+ true
2582026-06-22T19:15:51.792Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2592026-06-22T19:15:51.792Z+ EXPECTED_ZPOOL_COUNT=9
2602026-06-22T19:15:51.792Z+ echo 'expected number of zpools is 9'
2612026-06-22T19:15:51.792Zexpected number of zpools is 9
2622026-06-22T19:15:51.792Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2632026-06-22T19:15:51.792Z+ SILO_NAME=recovery
2642026-06-22T19:15:51.792Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2652026-06-22T19:15:51.792Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2662026-06-22T19:15:51.792Z+ sed -i~ '
2672026-06-22T19:15:51.792Z /^external_dns_ips/c\
2682026-06-22T19:15:51.792Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2692026-06-22T19:15:51.793Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2702026-06-22T19:15:51.793Z /^first/c\
2712026-06-22T19:15:51.793Zfirst = "10.151.2.164"
2722026-06-22T19:15:51.793Z /^last/c\
2732026-06-22T19:15:51.793Zlast = "10.151.2.173"
2742026-06-22T19:15:51.793Z }
2752026-06-22T19:15:51.793Z /^infra_ip_first/c\
2762026-06-22T19:15:51.793Zinfra_ip_first = "10.151.2.174"
2772026-06-22T19:15:51.793Z /^infra_ip_last/c\
2782026-06-22T19:15:51.793Zinfra_ip_last = "10.151.2.174"
2792026-06-22T19:15:51.793Z /^\[\[rack_network_config.ports/,/^$/ {
2802026-06-22T19:15:51.793Z /^routes/c\
2812026-06-22T19:15:51.793Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2822026-06-22T19:15:51.793Z /^addresses/c\
2832026-06-22T19:15:51.793Zaddresses = \[{address = {type = "static", ip_net = "10.151.2.174/24"} }\]
2842026-06-22T19:15:51.793Z }
2852026-06-22T19:15:51.793Z' pkg/config-rss.toml
2862026-06-22T19:15:51.793Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2872026-06-22T19:15:51.793Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2882026-06-22T19:15:51.793Z+++ pkg/config-rss.toml Mon Jun 22 19:15:51 2026
2892026-06-22T19:15:51.793Z@@ -33,7 +33,7 @@
2902026-06-22T19:15:51.793Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2912026-06-22T19:15:51.793Z # must be contained in one of the "internal services" IP Pool ranges listed
2922026-06-22T19:15:51.793Z # below.
2932026-06-22T19:15:51.793Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2942026-06-22T19:15:51.793Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2952026-06-22T19:15:51.793Z
2962026-06-22T19:15:51.793Z # Initial TLS certificates for the external API
2972026-06-22T19:15:51.793Z #
2982026-06-22T19:15:51.793Z@@ -69,8 +69,8 @@
2992026-06-22T19:15:51.793Z #
3002026-06-22T19:15:51.793Z # For more on this and what to put here, see docs/how-to-run.adoc.
3012026-06-22T19:15:51.793Z [[internal_services_ip_pool_ranges]]
3022026-06-22T19:15:51.793Z-first = "192.168.1.20"
3032026-06-22T19:15:51.793Z-last = "192.168.1.29"
3042026-06-22T19:15:51.794Z+first = "10.151.2.164"
3052026-06-22T19:15:51.794Z+last = "10.151.2.173"
3062026-06-22T19:15:51.794Z
3072026-06-22T19:15:51.794Z # TODO - this configuration is subject to change going forward. Ultimately these
3082026-06-22T19:15:51.794Z # parameters should be provided to the control plane via wicket, but we need to
3092026-06-22T19:15:51.794Z@@ -91,8 +91,8 @@
3102026-06-22T19:15:51.794Z # A range of IP addresses used by Boundary Services on the external network. In
3112026-06-22T19:15:51.794Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3122026-06-22T19:15:51.794Z # With softnpu, only one address is used.
3132026-06-22T19:15:51.794Z-infra_ip_first = "192.168.1.30"
3142026-06-22T19:15:51.794Z-infra_ip_last = "192.168.1.30"
3152026-06-22T19:15:51.794Z+infra_ip_first = "10.151.2.174"
3162026-06-22T19:15:51.794Z+infra_ip_last = "10.151.2.174"
3172026-06-22T19:15:51.794Z
3182026-06-22T19:15:51.794Z # Configurations for BGP routers to run on the scrimlets.
3192026-06-22T19:15:51.794Z bgp = []
3202026-06-22T19:15:51.794Z@@ -100,9 +100,9 @@
3212026-06-22T19:15:51.794Z # You can configure multiple uplinks by repeating the following stanza
3222026-06-22T19:15:51.794Z [[rack_network_config.ports]]
3232026-06-22T19:15:51.794Z # Routes associated with this port.
3242026-06-22T19:15:51.794Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3252026-06-22T19:15:51.818Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3262026-06-22T19:15:51.819Z # Addresses associated with this port.
3272026-06-22T19:15:51.819Z-addresses = [{address = {type = "static", ip_net = "192.168.1.30/24"}}]
3282026-06-22T19:15:51.819Z+addresses = [{address = {type = "static", ip_net = "10.151.2.174/24"} }]
3292026-06-22T19:15:51.819Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3302026-06-22T19:15:51.819Z port = "qsfp0"
3312026-06-22T19:15:51.819Z # The speed of this port.
3322026-06-22T19:15:51.819Z+ true
3332026-06-22T19:15:51.819Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3342026-06-22T19:15:51.819Za pkg/config-rss.toml 8K
3352026-06-22T19:15:51.819Za pkg/config.toml 6K
3362026-06-22T19:15:51.819Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3372026-06-22T19:15:51.819Z+ TLS_NAME=recovery.sys.oxide.test
3382026-06-22T19:15:51.819Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3392026-06-22T19:15:52.173Z....+...+......+.....+...+...+..........+........+......+.+...+..+...+....+...+..+.+.........+.....+.+.....+.......+..+++++++++++++++++++++++++++++++++++++++++++++*.+......+++++++++++++++++++++++++++++++++++++++++++++*.........+.+...+..+...+....+...+.....+.........+................+.....+.............+...............+.....+......+.........+.........+.......+.....+......+...+.+...........+............+..........+..+..........+........+..........+...+...+.....+............+...+.+......+.....+........................+...+.+...........+......+.......+...........+.+..............+.+.....+...+......+.+...........+...+..........+..+...+....+...+..+.+.................+...+.+......+..+.+..+.......+...+........+.+.....+...+......+.........+.........................+......+.........+........+..................+....+..+.+..+......+.......+...+.....+....+.....+............+.+...+..............+...+....+.........+.....+................+........+......+.+......+...+..+....+...+...+..+.......+...+..+.............+....................+......+....+...+...+................................+............+.......+......+........+...................+........+....+.........+..........................................+...............+...+.....+.........+...+....+...+............+...+.....+...+..........+...+.....+......+.........+.......+..................+.....+.......+.........+.....+.........+....+......+......+...+..+...............+...+.+......+.....+.........+............+.......+.....+..........+....................+.+.....+....+...........+.....................+.........+.......+++++
3402026-06-22T19:15:52.197Z.....+...+...............+....+...+............+......+..+.......+.....+...+++++++++++++++++++++++++++++++++++++++++++++*..+...+..+++++++++++++++++++++++++++++++++++++++++++++*..............+...+.....................................+.........+++++
3412026-06-22T19:15:52.221Z-----
3422026-06-22T19:15:52.222Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3432026-06-22T19:15:52.222Za pkg/initial-tls-cert.pem 2K
3442026-06-22T19:15:52.222Za pkg/initial-tls-key.pem 4K
3452026-06-22T19:15:52.222Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3462026-06-22T19:15:52.222Z+ rmdir pkg
3472026-06-22T19:15:52.246Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3482026-06-22T19:15:52.246Z+ pfexec mkdir -p /usr/oxide
3492026-06-22T19:15:52.246Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3502026-06-22T19:15:52.885Z+ pfexec chmod +x /usr/oxide/oxide
3512026-06-22T19:15:52.909Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3522026-06-22T19:16:11.482Z+ OMICRON_NO_UNINSTALL=1
3532026-06-22T19:16:11.482Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3542026-06-22T19:16:11.506ZLogging to: /opt/oxide/work/out/LOG
3552026-06-22T19:16:12.396Z
3562026-06-22T19:16:12.396Zreal 0.914976620
3572026-06-22T19:16:12.396Zuser 0.182976157
3582026-06-22T19:16:12.396Zsys 5.874154046
3592026-06-22T19:16:12.420Ztrap 0.000009398
3602026-06-22T19:16:12.420Ztflt 0.000039569
3612026-06-22T19:16:12.420Zdflt 0.000201783
3622026-06-22T19:16:12.420Zkflt 0.000001460
3632026-06-22T19:16:12.420Zlock 1:22.148402168
3642026-06-22T19:16:12.420Zslp 1.099545688
3652026-06-22T19:16:12.420Zlat 0.051679694
3662026-06-22T19:16:12.420Zstop 0.064404482
3672026-06-22T19:16:12.420Z+ retry=0
3682026-06-22T19:16:12.420Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3692026-06-22T19:16:12.420Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3702026-06-22T19:16:12.420Z+ [[ 0 -gt 30 ]]
3712026-06-22T19:16:12.420Z+ sleep 1
3722026-06-22T19:16:13.411Z+ retry=1
3732026-06-22T19:16:13.411Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3742026-06-22T19:16:13.435Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3752026-06-22T19:16:13.435Z+ [[ 1 -gt 30 ]]
3762026-06-22T19:16:13.435Z+ sleep 1
3772026-06-22T19:16:14.427Z+ retry=2
3782026-06-22T19:16:14.427Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3792026-06-22T19:16:14.458Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3802026-06-22T19:16:14.482Z+ [[ 2 -gt 30 ]]
3812026-06-22T19:16:14.482Z+ sleep 1
3822026-06-22T19:16:15.464Z+ retry=3
3832026-06-22T19:16:15.464Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3842026-06-22T19:16:16.479Zcurl: (28) Connection timed out after 1000 milliseconds
3852026-06-22T19:16:16.479Z+ [[ 3 -gt 30 ]]
3862026-06-22T19:16:16.479Z+ sleep 1
3872026-06-22T19:16:17.482Z+ retry=4
3882026-06-22T19:16:17.482Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3892026-06-22T19:16:18.495Zcurl: (28) Connection timed out after 1000 milliseconds
3902026-06-22T19:16:18.519Z+ [[ 4 -gt 30 ]]
3912026-06-22T19:16:18.519Z+ sleep 1
3922026-06-22T19:16:19.500Z+ retry=5
3932026-06-22T19:16:19.500Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3942026-06-22T19:16:20.518Zcurl: (28) Connection timed out after 1000 milliseconds
3952026-06-22T19:16:20.542Z+ [[ 5 -gt 30 ]]
3962026-06-22T19:16:20.542Z+ sleep 1
3972026-06-22T19:16:21.550Z+ retry=6
3982026-06-22T19:16:21.550Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3992026-06-22T19:16:22.566Zcurl: (28) Connection timed out after 1000 milliseconds
4002026-06-22T19:16:22.590Z+ [[ 6 -gt 30 ]]
4012026-06-22T19:16:22.590Z+ sleep 1
4022026-06-22T19:16:23.571Z+ retry=7
4032026-06-22T19:16:23.571Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4042026-06-22T19:16:24.585Zcurl: (28) Connection timed out after 1000 milliseconds
4052026-06-22T19:16:24.609Z+ [[ 7 -gt 30 ]]
4062026-06-22T19:16:24.609Z+ sleep 1
4072026-06-22T19:16:25.591Z+ retry=8
4082026-06-22T19:16:25.591Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4092026-06-22T19:16:26.604Zcurl: (28) Connection timed out after 1000 milliseconds
4102026-06-22T19:16:26.628Z+ [[ 8 -gt 30 ]]
4112026-06-22T19:16:26.628Z+ sleep 1
4122026-06-22T19:16:27.610Z+ retry=9
4132026-06-22T19:16:27.610Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4142026-06-22T19:16:28.633Zcurl: (28) Connection timed out after 1000 milliseconds
4152026-06-22T19:16:28.678Z+ [[ 9 -gt 30 ]]
4162026-06-22T19:16:28.678Z+ sleep 1
4172026-06-22T19:16:29.638Z+ retry=10
4182026-06-22T19:16:29.639Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4192026-06-22T19:16:30.669Zcurl: (28) Connection timed out after 1000 milliseconds
4202026-06-22T19:16:30.716Z+ [[ 10 -gt 30 ]]
4212026-06-22T19:16:30.717Z+ sleep 1
4222026-06-22T19:16:31.719Z+ retry=11
4232026-06-22T19:16:31.719Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4242026-06-22T19:16:32.748Zcurl: (28) Connection timed out after 1000 milliseconds
4252026-06-22T19:16:32.772Z+ [[ 11 -gt 30 ]]
4262026-06-22T19:16:32.772Z+ sleep 1
4272026-06-22T19:16:33.755Z+ retry=12
4282026-06-22T19:16:33.755Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4292026-06-22T19:16:34.765Zcurl: (28) Connection timed out after 1000 milliseconds
4302026-06-22T19:16:34.789Z+ [[ 12 -gt 30 ]]
4312026-06-22T19:16:34.789Z+ sleep 1
4322026-06-22T19:16:35.771Z+ retry=13
4332026-06-22T19:16:35.771Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4342026-06-22T19:16:36.221Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 438 ms: Could not connect to server
4352026-06-22T19:16:36.245Z+ [[ 13 -gt 30 ]]
4362026-06-22T19:16:36.245Z+ sleep 1
4372026-06-22T19:16:37.227Z+ retry=14
4382026-06-22T19:16:37.227Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4392026-06-22T19:16:37.251Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
4402026-06-22T19:16:37.276Zlocal v6:
4412026-06-22T19:16:37.276Zfe80::9010:7bff:fe72:e7c2
4422026-06-22T19:16:37.276Zfd68:dfa6:595c:dca1::1
4432026-06-22T19:16:37.276Zlocal v4:
4442026-06-22T19:16:37.276Z10.151.2.174
4452026-06-22T19:16:37.276Zrouter v6_idx:
4462026-06-22T19:16:37.276Zrouter v6_routes:
4472026-06-22T19:16:37.276Zrouter v4_idx:
4482026-06-22T19:16:37.276Zrouter v4_routes:
4492026-06-22T19:16:37.276Zresolver v4:
4502026-06-22T19:16:37.276Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
4512026-06-22T19:16:37.277Zresolver v6:
4522026-06-22T19:16:37.277Zfe80::2c02:51ff:fe6d:60fd -> 2e:02:51:6d:60:fd
4532026-06-22T19:16:37.277Zfe80::9010:7bff:fe72:e7c2 -> 92:10:7b:72:e7:c2
4542026-06-22T19:16:37.277Znat v4:
4552026-06-22T19:16:37.277Z10.151.2.169 0/16383 -> fd00:1122:3344:101::1 100/a8:40:25:ff:ef:53
4562026-06-22T19:16:37.277Znat_v6:
4572026-06-22T19:16:37.277Zattached subnet v4:
4582026-06-22T19:16:37.277Zattached subnet v6:
4592026-06-22T19:16:37.277Zport_mac:
4602026-06-22T19:16:37.277Z1: a8:40:25:51:38:fe
4612026-06-22T19:16:37.277Z2: a8:40:25:51:39:1e
4622026-06-22T19:16:37.277Zproxy arp:
4632026-06-22T19:16:37.277Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
4642026-06-22T19:16:37.277Z+ retry=0
4652026-06-22T19:16:37.277Z+++ zoneadm list -n
4662026-06-22T19:16:37.304Z+++ grep oxz_ntp
4672026-06-22T19:16:37.304Z++ pfexec svcs -z -Hostate oxide/ntp
4682026-06-22T19:16:37.304Zsvcs: zone '-Hostate': entity not found
4692026-06-22T19:16:37.304Z++ true
4702026-06-22T19:16:37.304Z+ [[ '' != online ]]
4712026-06-22T19:16:37.304Z+ [[ 0 -gt 60 ]]
4722026-06-22T19:16:37.304Z+ sleep 1
4732026-06-22T19:16:38.294Z+ retry=1
4742026-06-22T19:16:38.325Z+++ zoneadm list -n
4752026-06-22T19:16:38.325Z+++ grep oxz_ntp
4762026-06-22T19:16:38.325Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
4772026-06-22T19:16:38.359Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4782026-06-22T19:16:38.359Z++ true
4792026-06-22T19:16:38.359Z+ [[ '' != online ]]
4802026-06-22T19:16:38.359Z+ [[ 1 -gt 60 ]]
4812026-06-22T19:16:38.359Z+ sleep 1
4822026-06-22T19:16:39.327Z+ retry=2
4832026-06-22T19:16:39.354Z+++ zoneadm list -n
4842026-06-22T19:16:39.354Z+++ grep oxz_ntp
4852026-06-22T19:16:39.354Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
4862026-06-22T19:16:39.380Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4872026-06-22T19:16:39.380Z++ true
4882026-06-22T19:16:39.380Z+ [[ '' != online ]]
4892026-06-22T19:16:39.380Z+ [[ 2 -gt 60 ]]
4902026-06-22T19:16:39.380Z+ sleep 1
4912026-06-22T19:16:40.361Z+ retry=3
4922026-06-22T19:16:40.386Z+++ zoneadm list -n
4932026-06-22T19:16:40.386Z+++ grep oxz_ntp
4942026-06-22T19:16:40.386Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
4952026-06-22T19:16:40.410Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4962026-06-22T19:16:40.410Z++ true
4972026-06-22T19:16:40.410Z+ [[ '' != online ]]
4982026-06-22T19:16:40.410Z+ [[ 3 -gt 60 ]]
4992026-06-22T19:16:40.410Z+ sleep 1
5002026-06-22T19:16:41.392Z+ retry=4
5012026-06-22T19:16:41.416Z+++ zoneadm list -n
5022026-06-22T19:16:41.416Z+++ grep oxz_ntp
5032026-06-22T19:16:41.416Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5042026-06-22T19:16:41.440Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
5052026-06-22T19:16:41.440Z++ true
5062026-06-22T19:16:41.440Z+ [[ '' != online ]]
5072026-06-22T19:16:41.440Z+ [[ 4 -gt 60 ]]
5082026-06-22T19:16:41.440Z+ sleep 1
5092026-06-22T19:16:42.423Z+ retry=5
5102026-06-22T19:16:42.447Z+++ zoneadm list -n
5112026-06-22T19:16:42.447Z+++ grep oxz_ntp
5122026-06-22T19:16:42.447Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5132026-06-22T19:16:42.447Z+ [[ offline != online ]]
5142026-06-22T19:16:42.447Z+ [[ 5 -gt 60 ]]
5152026-06-22T19:16:42.447Z+ sleep 1
5162026-06-22T19:16:43.451Z+ retry=6
5172026-06-22T19:16:43.475Z+++ zoneadm list -n
5182026-06-22T19:16:43.475Z+++ grep oxz_ntp
5192026-06-22T19:16:43.475Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5202026-06-22T19:16:43.475Z+ [[ offline != online ]]
5212026-06-22T19:16:43.475Z+ [[ 6 -gt 60 ]]
5222026-06-22T19:16:43.475Z+ sleep 1
5232026-06-22T19:16:44.471Z+ retry=7
5242026-06-22T19:16:44.495Z+++ zoneadm list -n
5252026-06-22T19:16:44.495Z+++ grep oxz_ntp
5262026-06-22T19:16:44.495Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5272026-06-22T19:16:44.495Z+ [[ offline != online ]]
5282026-06-22T19:16:44.495Z+ [[ 7 -gt 60 ]]
5292026-06-22T19:16:44.496Z+ sleep 1
5302026-06-22T19:16:45.552Z+ retry=8
5312026-06-22T19:16:45.580Z+++ zoneadm list -n
5322026-06-22T19:16:45.580Z+++ grep oxz_ntp
5332026-06-22T19:16:45.580Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5342026-06-22T19:16:45.580Z+ [[ offline != online ]]
5352026-06-22T19:16:45.580Z+ [[ 8 -gt 60 ]]
5362026-06-22T19:16:45.580Z+ sleep 1
5372026-06-22T19:16:46.525Z+ retry=9
5382026-06-22T19:16:46.567Z+++ zoneadm list -n
5392026-06-22T19:16:46.567Z+++ grep oxz_ntp
5402026-06-22T19:16:46.567Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5412026-06-22T19:16:46.568Z+ [[ offline != online ]]
5422026-06-22T19:16:46.568Z+ [[ 9 -gt 60 ]]
5432026-06-22T19:16:46.568Z+ sleep 1
5442026-06-22T19:16:47.556Z+ retry=10
5452026-06-22T19:16:47.603Z+++ zoneadm list -n
5462026-06-22T19:16:47.603Z+++ grep oxz_ntp
5472026-06-22T19:16:47.603Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5482026-06-22T19:16:47.603Z+ [[ offline != online ]]
5492026-06-22T19:16:47.603Z+ [[ 10 -gt 60 ]]
5502026-06-22T19:16:47.603Z+ sleep 1
5512026-06-22T19:16:48.565Z+ retry=11
5522026-06-22T19:16:48.593Z+++ zoneadm list -n
5532026-06-22T19:16:48.593Z+++ grep oxz_ntp
5542026-06-22T19:16:48.593Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5552026-06-22T19:16:48.593Z+ [[ offline != online ]]
5562026-06-22T19:16:48.593Z+ [[ 11 -gt 60 ]]
5572026-06-22T19:16:48.594Z+ sleep 1
5582026-06-22T19:16:49.590Z+ retry=12
5592026-06-22T19:16:49.614Z+++ zoneadm list -n
5602026-06-22T19:16:49.614Z+++ grep oxz_ntp
5612026-06-22T19:16:49.614Z++ pfexec svcs -z oxz_ntp_fcbe90fa-4ab4-4532-b388-3a36dd9230f4 -Hostate oxide/ntp
5622026-06-22T19:16:49.614Z+ [[ online != online ]]
5632026-06-22T19:16:49.614Z+ echo 'Waited for chrony: 12s'
5642026-06-22T19:16:49.614Z+ retry=0
5652026-06-22T19:16:49.614ZWaited for chrony: 12s
5662026-06-22T19:16:49.614Z+ zoneadm list
5672026-06-22T19:16:49.614Z+ grep nexus
5682026-06-22T19:16:49.638Z+ [[ 0 -gt 300 ]]
5692026-06-22T19:16:49.638Z+ sleep 1
5702026-06-22T19:16:50.622Z+ retry=1
5712026-06-22T19:16:50.645Z+ zoneadm list
5722026-06-22T19:16:50.645Z+ grep nexus
5732026-06-22T19:16:50.646Z+ [[ 1 -gt 300 ]]
5742026-06-22T19:16:50.646Z+ sleep 1
5752026-06-22T19:16:51.633Z+ retry=2
5762026-06-22T19:16:51.657Z+ zoneadm list
5772026-06-22T19:16:51.657Z+ grep nexus
5782026-06-22T19:16:51.657Z+ [[ 2 -gt 300 ]]
5792026-06-22T19:16:51.657Z+ sleep 1
5802026-06-22T19:16:52.646Z+ retry=3
5812026-06-22T19:16:52.670Z+ zoneadm list
5822026-06-22T19:16:52.670Z+ grep nexus
5832026-06-22T19:16:52.670Z+ [[ 3 -gt 300 ]]
5842026-06-22T19:16:52.670Z+ sleep 1
5852026-06-22T19:16:53.663Z+ retry=4
5862026-06-22T19:16:53.688Z+ zoneadm list
5872026-06-22T19:16:53.688Z+ grep nexus
5882026-06-22T19:16:53.688Z+ [[ 4 -gt 300 ]]
5892026-06-22T19:16:53.688Z+ sleep 1
5902026-06-22T19:16:54.676Z+ retry=5
5912026-06-22T19:16:54.700Z+ zoneadm list
5922026-06-22T19:16:54.700Z+ grep nexus
5932026-06-22T19:16:54.700Z+ [[ 5 -gt 300 ]]
5942026-06-22T19:16:54.700Z+ sleep 1
5952026-06-22T19:16:55.691Z+ retry=6
5962026-06-22T19:16:55.715Z+ zoneadm list
5972026-06-22T19:16:55.715Z+ grep nexus
5982026-06-22T19:16:55.715Z+ [[ 6 -gt 300 ]]
5992026-06-22T19:16:55.715Z+ sleep 1
6002026-06-22T19:16:56.745Z+ retry=7
6012026-06-22T19:16:56.848Z+ zoneadm list
6022026-06-22T19:16:56.849Z+ grep nexus
6032026-06-22T19:16:56.849Z+ [[ 7 -gt 300 ]]
6042026-06-22T19:16:56.849Z+ sleep 1
6052026-06-22T19:16:57.757Z+ retry=8
6062026-06-22T19:16:57.796Z+ zoneadm list
6072026-06-22T19:16:57.796Z+ grep nexus
6082026-06-22T19:16:57.796Z+ [[ 8 -gt 300 ]]
6092026-06-22T19:16:57.796Z+ sleep 1
6102026-06-22T19:16:58.775Z+ retry=9
6112026-06-22T19:16:58.804Z+ zoneadm list
6122026-06-22T19:16:58.804Z+ grep nexus
6132026-06-22T19:16:58.804Z+ [[ 9 -gt 300 ]]
6142026-06-22T19:16:58.804Z+ sleep 1
6152026-06-22T19:16:59.791Z+ retry=10
6162026-06-22T19:16:59.815Z+ zoneadm list
6172026-06-22T19:16:59.815Z+ grep nexus
6182026-06-22T19:16:59.815Z+ [[ 10 -gt 300 ]]
6192026-06-22T19:16:59.815Z+ sleep 1
6202026-06-22T19:17:00.908Z+ retry=11
6212026-06-22T19:17:01.070Z+ zoneadm list
6222026-06-22T19:17:01.070Z+ grep nexus
6232026-06-22T19:17:01.070Z+ [[ 11 -gt 300 ]]
6242026-06-22T19:17:01.070Z+ sleep 1
6252026-06-22T19:17:01.983Z+ retry=12
6262026-06-22T19:17:02.008Z+ zoneadm list
6272026-06-22T19:17:02.008Z+ grep nexus
6282026-06-22T19:17:02.008Z+ [[ 12 -gt 300 ]]
6292026-06-22T19:17:02.008Z+ sleep 1
6302026-06-22T19:17:03.002Z+ retry=13
6312026-06-22T19:17:03.026Z+ zoneadm list
6322026-06-22T19:17:03.026Z+ grep nexus
6332026-06-22T19:17:03.026Z+ [[ 13 -gt 300 ]]
6342026-06-22T19:17:03.027Z+ sleep 1
6352026-06-22T19:17:04.018Z+ retry=14
6362026-06-22T19:17:04.044Z+ zoneadm list
6372026-06-22T19:17:04.044Z+ grep nexus
6382026-06-22T19:17:04.044Z+ [[ 14 -gt 300 ]]
6392026-06-22T19:17:04.044Z+ sleep 1
6402026-06-22T19:17:05.032Z+ retry=15
6412026-06-22T19:17:05.056Z+ zoneadm list
6422026-06-22T19:17:05.056Z+ grep nexus
6432026-06-22T19:17:05.056Z+ [[ 15 -gt 300 ]]
6442026-06-22T19:17:05.056Z+ sleep 1
6452026-06-22T19:17:06.055Z+ retry=16
6462026-06-22T19:17:06.083Z+ zoneadm list
6472026-06-22T19:17:06.083Z+ grep nexus
6482026-06-22T19:17:06.083Z+ [[ 16 -gt 300 ]]
6492026-06-22T19:17:06.083Z+ sleep 1
6502026-06-22T19:17:07.070Z+ retry=17
6512026-06-22T19:17:07.094Z+ zoneadm list
6522026-06-22T19:17:07.094Z+ grep nexus
6532026-06-22T19:17:07.094Z+ [[ 17 -gt 300 ]]
6542026-06-22T19:17:07.094Z+ sleep 1
6552026-06-22T19:17:08.084Z+ retry=18
6562026-06-22T19:17:08.110Z+ zoneadm list
6572026-06-22T19:17:08.110Z+ grep nexus
6582026-06-22T19:17:08.110Z+ [[ 18 -gt 300 ]]
6592026-06-22T19:17:08.110Z+ sleep 1
6602026-06-22T19:17:09.101Z+ retry=19
6612026-06-22T19:17:09.126Z+ zoneadm list
6622026-06-22T19:17:09.126Z+ grep nexus
6632026-06-22T19:17:09.126Z+ [[ 19 -gt 300 ]]
6642026-06-22T19:17:09.126Z+ sleep 1
6652026-06-22T19:17:10.114Z+ retry=20
6662026-06-22T19:17:10.139Z+ zoneadm list
6672026-06-22T19:17:10.139Z+ grep nexus
6682026-06-22T19:17:10.139Z+ [[ 20 -gt 300 ]]
6692026-06-22T19:17:10.139Z+ sleep 1
6702026-06-22T19:17:11.131Z+ retry=21
6712026-06-22T19:17:11.155Z+ zoneadm list
6722026-06-22T19:17:11.155Z+ grep nexus
6732026-06-22T19:17:11.155Z+ [[ 21 -gt 300 ]]
6742026-06-22T19:17:11.155Z+ sleep 1
6752026-06-22T19:17:12.147Z+ retry=22
6762026-06-22T19:17:12.170Z+ zoneadm list
6772026-06-22T19:17:12.170Z+ grep nexus
6782026-06-22T19:17:12.195Z+ [[ 22 -gt 300 ]]
6792026-06-22T19:17:12.195Z+ sleep 1
6802026-06-22T19:17:13.178Z+ retry=23
6812026-06-22T19:17:13.203Z+ zoneadm list
6822026-06-22T19:17:13.203Z+ grep nexus
6832026-06-22T19:17:13.203Z+ [[ 23 -gt 300 ]]
6842026-06-22T19:17:13.203Z+ sleep 1
6852026-06-22T19:17:14.192Z+ retry=24
6862026-06-22T19:17:14.216Z+ zoneadm list
6872026-06-22T19:17:14.216Z+ grep nexus
6882026-06-22T19:17:14.216Z+ [[ 24 -gt 300 ]]
6892026-06-22T19:17:14.216Z+ sleep 1
6902026-06-22T19:17:15.211Z+ retry=25
6912026-06-22T19:17:15.262Z+ zoneadm list
6922026-06-22T19:17:15.262Z+ grep nexus
6932026-06-22T19:17:15.262Z+ [[ 25 -gt 300 ]]
6942026-06-22T19:17:15.262Z+ sleep 1
6952026-06-22T19:17:16.229Z+ retry=26
6962026-06-22T19:17:16.253Z+ zoneadm list
6972026-06-22T19:17:16.253Z+ grep nexus
6982026-06-22T19:17:16.253Z+ [[ 26 -gt 300 ]]
6992026-06-22T19:17:16.253Z+ sleep 1
7002026-06-22T19:17:17.243Z+ retry=27
7012026-06-22T19:17:17.267Z+ zoneadm list
7022026-06-22T19:17:17.267Z+ grep nexus
7032026-06-22T19:17:17.267Z+ [[ 27 -gt 300 ]]
7042026-06-22T19:17:17.267Z+ sleep 1
7052026-06-22T19:17:18.257Z+ retry=28
7062026-06-22T19:17:18.283Z+ zoneadm list
7072026-06-22T19:17:18.283Z+ grep nexus
7082026-06-22T19:17:18.283Z+ [[ 28 -gt 300 ]]
7092026-06-22T19:17:18.283Z+ sleep 1
7102026-06-22T19:17:19.269Z+ retry=29
7112026-06-22T19:17:19.293Z+ zoneadm list
7122026-06-22T19:17:19.293Z+ grep nexus
7132026-06-22T19:17:19.293Z+ [[ 29 -gt 300 ]]
7142026-06-22T19:17:19.293Z+ sleep 1
7152026-06-22T19:17:20.305Z+ retry=30
7162026-06-22T19:17:20.328Z+ zoneadm list
7172026-06-22T19:17:20.328Z+ grep nexus
7182026-06-22T19:17:20.328Z+ [[ 30 -gt 300 ]]
7192026-06-22T19:17:20.329Z+ sleep 1
7202026-06-22T19:17:21.297Z+ retry=31
7212026-06-22T19:17:21.321Z+ zoneadm list
7222026-06-22T19:17:21.321Z+ grep nexus
7232026-06-22T19:17:21.321Z+ [[ 31 -gt 300 ]]
7242026-06-22T19:17:21.321Z+ sleep 1
7252026-06-22T19:17:22.312Z+ retry=32
7262026-06-22T19:17:22.336Z+ zoneadm list
7272026-06-22T19:17:22.337Z+ grep nexus
7282026-06-22T19:17:22.337Z+ [[ 32 -gt 300 ]]
7292026-06-22T19:17:22.337Z+ sleep 1
7302026-06-22T19:17:23.325Z+ retry=33
7312026-06-22T19:17:23.349Z+ zoneadm list
7322026-06-22T19:17:23.349Z+ grep nexus
7332026-06-22T19:17:23.349Z+ [[ 33 -gt 300 ]]
7342026-06-22T19:17:23.349Z+ sleep 1
7352026-06-22T19:17:24.338Z+ retry=34
7362026-06-22T19:17:24.362Z+ zoneadm list
7372026-06-22T19:17:24.362Z+ grep nexus
7382026-06-22T19:17:24.362Z+ [[ 34 -gt 300 ]]
7392026-06-22T19:17:24.362Z+ sleep 1
7402026-06-22T19:17:25.354Z+ retry=35
7412026-06-22T19:17:25.378Z+ zoneadm list
7422026-06-22T19:17:25.378Z+ grep nexus
7432026-06-22T19:17:25.378Z+ [[ 35 -gt 300 ]]
7442026-06-22T19:17:25.378Z+ sleep 1
7452026-06-22T19:17:26.367Z+ retry=36
7462026-06-22T19:17:26.391Z+ zoneadm list
7472026-06-22T19:17:26.391Z+ grep nexus
7482026-06-22T19:17:26.392Z+ [[ 36 -gt 300 ]]
7492026-06-22T19:17:26.392Z+ sleep 1
7502026-06-22T19:17:27.385Z+ retry=37
7512026-06-22T19:17:27.461Z+ zoneadm list
7522026-06-22T19:17:27.461Z+ grep nexus
7532026-06-22T19:17:27.461Z+ [[ 37 -gt 300 ]]
7542026-06-22T19:17:27.461Z+ sleep 1
7552026-06-22T19:17:28.398Z+ retry=38
7562026-06-22T19:17:28.422Z+ zoneadm list
7572026-06-22T19:17:28.422Z+ grep nexus
7582026-06-22T19:17:28.422Z+ [[ 38 -gt 300 ]]
7592026-06-22T19:17:28.422Z+ sleep 1
7602026-06-22T19:17:29.410Z+ retry=39
7612026-06-22T19:17:29.434Z+ zoneadm list
7622026-06-22T19:17:29.434Z+ grep nexus
7632026-06-22T19:17:29.434Z+ [[ 39 -gt 300 ]]
7642026-06-22T19:17:29.434Z+ sleep 1
7652026-06-22T19:17:30.425Z+ retry=40
7662026-06-22T19:17:30.449Z+ zoneadm list
7672026-06-22T19:17:30.449Z+ grep nexus
7682026-06-22T19:17:30.449Z+ [[ 40 -gt 300 ]]
7692026-06-22T19:17:30.449Z+ sleep 1
7702026-06-22T19:17:31.446Z+ retry=41
7712026-06-22T19:17:31.475Z+ zoneadm list
7722026-06-22T19:17:31.475Z+ grep nexus
7732026-06-22T19:17:31.475Z+ [[ 41 -gt 300 ]]
7742026-06-22T19:17:31.475Z+ sleep 1
7752026-06-22T19:17:32.479Z+ retry=42
 
7762026-06-22T19:19:05.417Zsock console login:
7772026-06-22T19:19:05.468Zpanic[cpu0]/thread=fffffe00f3cf2c20: pageout_deadman: stuck pushing the same page for 90 seconds (freemem is 176283)
7782026-06-22T19:19:05.513Zfffffe00f3cf2a60 genunix:pageout_deadman+62 ()
7792026-06-22T19:19:05.644Zfffffe00f3cf2ad0 genunix:clock+7b3 ()
7802026-06-22T19:19:05.712Zfffffe00f3cf2b60 genunix:cyclic_softint+e1 ()
7812026-06-22T19:19:05.746Zfffffe00f3cf2b80 unix:cbe_softclock+23 ()
7822026-06-22T19:19:05.788Zfffffe00f3cf2bd0 unix:av_dispatch_softvect+72 ()
7832026-06-22T19:19:05.957Zfffffe00f3cf2c00 apix:apix_dispatch_softint+35 ()
7842026-06-22T19:19:05.991Zfffffe00f3c05970 unix:switch_sp_and_call+15 ()
7852026-06-22T19:19:06.025Zfffffe00f3c059c0 apix:apix_do_softint+5a ()
7862026-06-22T19:19:06.058Zfffffe00f3c05a20 apix:apix_do_interrupt+2bf ()
7872026-06-22T19:19:06.091Zfffffe00f3c05a30 unix:cmnint+c3 ()
7882026-06-22T19:19:06.125Zfffffe00f3c05ba0 unix:acpi_cpu_cstate+133 ()
7892026-06-22T19:19:06.159Zfffffe00f3c05bd0 unix:cpu_acpi_idle+75 ()
7902026-06-22T19:19:06.192Zfffffe00f3c05be0 unix:cpu_idle_adaptive+19 ()
7912026-06-22T19:19:06.225Zfffffe00f3c05c00 unix:idle+a8 ()
7922026-06-22T19:19:06.258Zfffffe00f3c05c10 unix:thread_start+b ()
7932026-06-22T19:19:06.292Zskipping system dump - no dump device configured
7942026-06-22T19:19:06.348Zrebooting...
7952026-06-22T19:20:40.369Z^[ciPXE initialising devices...ok
7962026-06-22T19:20:40.431Z^[[0m
7972026-06-22T19:20:40.478Z^[[1miPXE 1.21.1+ (g8bff5)^[[0m -- Open Source Network Boot Firmwar
7982026-06-22T19:20:40.512Z-- ^[[36mhttp://ipxe.org^[[0m
7992026-06-22T19:20:40.547ZFeatures: DNS HTTP iSCSI TFTP SRP AoE ELF MBOOT PXE bzImage Menu PXEXT
8002026-06-22T19:20:42.346Znet0: 18:c0:4d:0d:a0:2a using i350 on 0000:c3:00.0 (Ethernet) [open]
8012026-06-22T19:20:42.383Z[Link:up, TX:0 TXE:1 RX:0 RXE:0]
8022026-06-22T19:20:42.416Z[TXE: 1 x "Network unreachable (http://ipxe.org/28086011)"]
8032026-06-22T19:20:45.504ZConfiguring (net0 18:c0:4d:0d:a0:2a)...... ok
8042026-06-22T19:20:45.542Znet0: 10.151.2.100/255.255.255.0 gw 10.151.2.1
8052026-06-22T19:20:45.579ZFilename: http://172.20.0.5:9969/ipxe/sock
8062026-06-22T19:20:45.612Zhttp://172.20.0.5:9969/ipxe/sock... ok
8072026-06-22T19:20:45.646Zsock : 555 bytes [script]
8082026-06-22T19:22:42.729Zworker failed without completing job
8092026-06-22T19:22:42.732Ztask 2 was incomplete, marked failed