01KF64P68CP92ZCKX5F9HP8F0R: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.37",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KF64Q0XK3PZWJWACHVZW6AM8

Tags:

Output:

SEQ GLOBAL TIME DETAILS
12026-01-17T15:01:27.825Zjob dependencies complete; ready to run (waiting for 50 m 28 s)
22026-01-17T15:02:33.075Zjob assigned to worker 01KF67KM6VHEQJ8S1H85Q4PXRE [factory lab, sock/15673] (queued for 1 m 5 s)
32026-01-17T15:02:33.136Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-01-17T15:02:33.185Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52026-01-17T15:02:33.219Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62026-01-17T15:02:33.253Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72026-01-17T15:02:33.327Zhttp://172.20.0.5:9969/postboot/sock... ok
82026-01-17T15:02:33.361ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92026-01-17T15:02:33.394ZWARNING: cpu0: failed to update microcode from version 0xa001133 to 0xa0011de
102026-01-17T15:02:33.428ZWARNING: cpu0: using older fallback microcode; update the system firmware
112026-01-17T15:02:33.461Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d5
122026-01-17T15:02:33.499ZOxide Helios Version helios-2.0.23698 64-bit
132026-01-17T15:02:33.534ZConfiguring devices.
142026-01-17T15:02:33.568Z##################################################
152026-01-17T15:02:33.601Z#### ############# ##
162026-01-17T15:02:33.635Z### ### ############ ##
172026-01-17T15:02:33.668Z## ### # ## ### ## Oxide Computer Company ##
182026-01-17T15:02:33.702Z## ## ## ### # ### ##
192026-01-17T15:02:33.735Z## # ### #### #### This Station Under ##
202026-01-17T15:02:33.769Z### ### #### # ### Computer Control ##
212026-01-17T15:02:33.802Z#### #### ### ## ##
222026-01-17T15:02:33.835Z##################################################
232026-01-17T15:02:33.869ZHostname: sock
242026-01-17T15:02:33.902ZLoading smf(7) service descriptions:
252026-01-17T15:02:34.177Zdownloading input: /input/package/work/package.tar.gz
262026-01-17T15:09:24.459Zdownloaded input: /input/package/work/package.tar.gz
 
272026-01-17T15:09:24.459Zstarting task 0: "setup"
282026-01-17T15:09:24.483Z++ uname -s
292026-01-17T15:09:24.483Z+ kern=SunOS
302026-01-17T15:09:24.483Z+ build_user=build
312026-01-17T15:09:24.484Z+ build_uid=12345
322026-01-17T15:09:24.484Z+ work_dir=/work
332026-01-17T15:09:24.484Z+ input_dir=/input
342026-01-17T15:09:24.484Z+ [[ 0 == 12345 ]]
352026-01-17T15:09:24.484Z+ case "$kern" in
362026-01-17T15:09:24.484Z+ groupadd -g 12345 build
372026-01-17T15:09:24.484Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-01-17T15:09:26.445Z+ zfs create -o mountpoint=/work rpool/work
392026-01-17T15:09:26.478Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-01-17T15:09:26.502Z+ home_fs=zfs
412026-01-17T15:09:26.502Z+ [[ zfs == autofs ]]
422026-01-17T15:09:26.502Z+ mkdir -p /home/build
432026-01-17T15:09:26.502Z+ chown build:build /home/build /work
442026-01-17T15:09:28.487Z+ chmod 0700 /home/build /work
452026-01-17T15:09:28.510Zprocess exited: duration 4064 ms, exit code 0
 
462026-01-17T15:09:28.558Zstarting task 1: "authentication"
472026-01-17T15:09:28.604Zprocess exited: duration 45 ms, exit code 0
 
482026-01-17T15:09:28.651Zstarting task 2: "build"
492026-01-17T15:09:28.674Z+ trap _exit_trap EXIT
502026-01-17T15:09:28.674Z+ OPTE_COMMIT=
512026-01-17T15:09:28.674Z+ [[ x != \x ]]
522026-01-17T15:09:28.675Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
532026-01-17T15:09:28.768Zsecpolicy_fs_linkdir: 0x55 = 0x55
542026-01-17T15:09:28.768Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
552026-01-17T15:09:28.768Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
562026-01-17T15:09:28.768Zsecpolicy_fs_linkdir+3: 0 = 0xe5
572026-01-17T15:09:28.768Zsecpolicy_fs_linkdir+4: 0 = 0xb8
582026-01-17T15:09:28.769Zsecpolicy_fs_linkdir+5: 0 = 0x1
592026-01-17T15:09:28.769Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
602026-01-17T15:09:28.769Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
612026-01-17T15:09:28.769Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
622026-01-17T15:09:28.769Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
632026-01-17T15:09:28.769Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
642026-01-17T15:09:28.792Z+ [[ -d /opt/oxide ]]
652026-01-17T15:09:28.792Z+ pfexec mv /opt/oxide /opt/oxide-underneath
662026-01-17T15:09:28.792Z+ pfexec mkdir /opt/oxide
672026-01-17T15:09:28.792Z+ pfexec mount -F tmpfs -O swap /opt/oxide
682026-01-17T15:09:28.792Z+ [[ -d /opt/oxide-underneath ]]
692026-01-17T15:09:28.792Z+ cd /opt/oxide-underneath
702026-01-17T15:09:28.792Z+ pfexec tar ceEp@/f - .
712026-01-17T15:09:28.792Z+ cd /opt/oxide
722026-01-17T15:09:28.792Z+ pfexec tar xveEp@/f -
732026-01-17T15:09:28.830Zx ., 0 bytes, 0 tape blocks
742026-01-17T15:09:28.830Zx ./opte, 0 bytes, 0 tape blocks
752026-01-17T15:09:28.830Zx ./opte/bin, 0 bytes, 0 tape blocks
762026-01-17T15:09:28.830Zx ./opte/bin/opteadm, 21006608 bytes, 41029 tape blocks
772026-01-17T15:09:28.854Z+ pfexec rm -rf /opt/oxide-underneath
782026-01-17T15:09:28.854Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
792026-01-17T15:09:28.877Z+ pfexec mkdir /opt/oxide/work
802026-01-17T15:09:28.877Z+ pfexec chown build:build /opt/oxide/work
812026-01-17T15:09:28.877Z+ cd /opt/oxide/work
822026-01-17T15:09:28.877Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
832026-01-17T15:09:50.492ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
842026-01-17T15:09:50.492Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
852026-01-17T15:09:50.492Zx out/target/test, 107 bytes, 1 tape blocks
862026-01-17T15:09:50.492Zx out/npuzone/npuzone, 23699288 bytes, 46288 tape blocks
872026-01-17T15:09:50.493Zx package-manifest.toml, 34403 bytes, 68 tape blocks
882026-01-17T15:09:50.493Zx smf/sled-agent/non-gimlet/config.toml, 5304 bytes, 11 tape blocks
892026-01-17T15:09:50.493Zx target/release/omicron-package, 43940384 bytes, 85822 tape blocks
902026-01-17T15:09:50.493Zx target/release/xtask, 11738264 bytes, 22927 tape blocks
912026-01-17T15:09:50.493Zx target/debug/bootstrap, 325698416 bytes, 636130 tape blocks
922026-01-17T15:09:50.493Zx tests/bootstrap-ec8a24da95b8f34e, 31868752 bytes, 62244 tape blocks
932026-01-17T15:09:50.493Zx tests/commtest-c6b535757a2dc792, 32732592 bytes, 63931 tape blocks
942026-01-17T15:09:50.493Zx tests/dhcp_server-527c36bd597fe3b3, 31868768 bytes, 62244 tape blocks
952026-01-17T15:09:50.493Zx tests/end_to_end_tests-b7fb91788d352d5d, 371040352 bytes, 724689 tape blocks
962026-01-17T15:09:50.493Zx out/clickhouse.tar.gz, 328069159 bytes, 640761 tape blocks
972026-01-17T15:09:50.493Zx out/clickhouse_keeper.tar.gz, 310651512 bytes, 606742 tape blocks
982026-01-17T15:09:50.493Zx out/clickhouse_server.tar.gz, 328067766 bytes, 640758 tape blocks
992026-01-17T15:09:50.493Zx out/cockroachdb.tar.gz, 171909895 bytes, 335762 tape blocks
1002026-01-17T15:09:50.493Zx out/crucible-pantry-zone.tar.gz, 45571093 bytes, 89007 tape blocks
1012026-01-17T15:09:50.493Zx out/crucible-utils.tar, 18169344 bytes, 35487 tape blocks
1022026-01-17T15:09:50.493Zx out/crucible-zone.tar.gz, 60048449 bytes, 117283 tape blocks
1032026-01-17T15:09:50.493Zx out/external-dns.tar.gz, 54846154 bytes, 107122 tape blocks
1042026-01-17T15:09:50.493Zx out/internal-dns.tar.gz, 54845753 bytes, 107121 tape blocks
1052026-01-17T15:09:50.494Zx out/mg-ddm-gz.tar, 116893184 bytes, 228307 tape blocks
1062026-01-17T15:09:50.494Zx out/nexus.tar.gz, 204138639 bytes, 398709 tape blocks
1072026-01-17T15:09:50.494Zx out/ntp.tar.gz, 35684546 bytes, 69697 tape blocks
1082026-01-17T15:09:50.494Zx out/omicron-gateway-softnpu.tar.gz, 30834082 bytes, 60223 tape blocks
1092026-01-17T15:09:50.494Zx out/omicron-sled-agent.tar, 199819264 bytes, 390272 tape blocks
1102026-01-17T15:09:50.494Zx out/overlay.tar.gz, 3030 bytes, 6 tape blocks
1112026-01-17T15:09:50.494Zx out/oximeter.tar.gz, 68086438 bytes, 132982 tape blocks
1122026-01-17T15:09:50.494Zx out/oxlog.tar, 4036608 bytes, 7884 tape blocks
1132026-01-17T15:09:50.494Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1142026-01-17T15:09:50.494Zx out/propolis-server.tar.gz, 76312333 bytes, 149048 tape blocks
1152026-01-17T15:09:50.494Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1162026-01-17T15:09:50.494Zx out/switch-softnpu.tar.gz, 392147927 bytes, 765914 tape blocks
1172026-01-17T15:09:50.518Z
1182026-01-17T15:09:50.519Zreal 21.607809169
1192026-01-17T15:09:50.519Zuser 19.330622993
1202026-01-17T15:09:50.519Zsys 3.820169183
1212026-01-17T15:09:50.519Ztrap 0.000931238
1222026-01-17T15:09:50.519Ztflt 0.000000000
1232026-01-17T15:09:50.519Zdflt 0.000000000
1242026-01-17T15:09:50.519Zkflt 0.000000000
1252026-01-17T15:09:50.519Zlock 0.000000000
1262026-01-17T15:09:50.519Zslp 19.513546764
1272026-01-17T15:09:50.519Zlat 0.548469060
1282026-01-17T15:09:50.519Zstop 0.000128863
1292026-01-17T15:09:50.519Z+ source .github/buildomat/ci-env.sh
1302026-01-17T15:09:50.519Z++ export CARGO_TERM_COLOR=always
1312026-01-17T15:09:50.519Z++ CARGO_TERM_COLOR=always
1322026-01-17T15:09:50.519Z+ bmat address ls
1332026-01-17T15:09:50.519ZNAME CIDR FIRST LAST COUNT
1342026-01-17T15:09:50.519Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1352026-01-17T15:09:50.519Z++ bmat address ls -f extra -Ho first,last,count
1362026-01-17T15:09:50.519Z+ set -- 10.151.2.164 10.151.2.227 64
1372026-01-17T15:09:50.519Z+ EXTRA_IP_START=10.151.2.164
1382026-01-17T15:09:50.519Z+ EXTRA_IP_END=10.151.2.227
1392026-01-17T15:09:50.519Z+ EXTRA_IP_COUNT=64
1402026-01-17T15:09:50.519Z+ (( EXTRA_IP_COUNT >= 32 ))
1412026-01-17T15:09:50.519Z+ EXTRA_IP_BASE=10.151.2
1422026-01-17T15:09:50.519Z+ EXTRA_IP_FOCTET=164
1432026-01-17T15:09:50.520Z+ EXTRA_IP_LOCTET=227
1442026-01-17T15:09:50.520Z+ SERVICE_IP_POOL_START=10.151.2.164
1452026-01-17T15:09:50.520Z+ SERVICE_IP_POOL_END=10.151.2.173
1462026-01-17T15:09:50.520Z+ DNS_IP1=10.151.2.164
1472026-01-17T15:09:50.520Z+ DNS_IP2=10.151.2.165
1482026-01-17T15:09:50.520Z+ UPLINK_IP=10.151.2.174
1492026-01-17T15:09:50.520Z+ IPPOOL_START=10.151.2.175
1502026-01-17T15:09:50.520Z+ IPPOOL_END=10.151.2.227
1512026-01-17T15:09:50.520Z++ ipadm show-addr -po type,addr
1522026-01-17T15:09:50.520Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1532026-01-17T15:09:50.520Z+ GATEWAY_IP=10.151.2.100
1542026-01-17T15:09:50.520Z+ [[ -n 10.151.2.100 ]]
1552026-01-17T15:09:50.520Z+ ping -s 10.151.2.100 56 1
1562026-01-17T15:09:50.520ZPING 10.151.2.100: 56 data bytes
1572026-01-17T15:09:50.520Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.010 ms
1582026-01-17T15:09:50.520Z
1592026-01-17T15:09:50.520Z----10.151.2.100 PING Statistics----
1602026-01-17T15:09:50.520Z1 packets transmitted, 1 packets received, 0% packet loss
1612026-01-17T15:09:50.520Zround-trip (ms) min/avg/max/stddev = 0.010/0.010/0.010/-nan
1622026-01-17T15:09:50.520Z++ arp -an
1632026-01-17T15:09:50.520Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1642026-01-17T15:09:50.544Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1652026-01-17T15:09:50.544Z+ routeadm -e ipv4-forwarding -u
1662026-01-17T15:09:50.620Z+ PXA_START=10.151.2.164
1672026-01-17T15:09:50.620Z+ PXA_END=10.151.2.227
1682026-01-17T15:09:50.644Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
1692026-01-17T15:09:50.797Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1702026-01-17T15:09:50.820ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1712026-01-17T15:09:50.871Zcreating virtual hardware
1722026-01-17T15:09:50.894Zcreating /scratch/m2_0.vdev
1732026-01-17T15:09:50.894Zcreating /scratch/m2_1.vdev
1742026-01-17T15:09:50.895Zcreating /scratch/u2_0.vdev
1752026-01-17T15:09:50.895Zcreating /scratch/u2_1.vdev
1762026-01-17T15:09:50.895Zcreating /scratch/u2_2.vdev
1772026-01-17T15:09:50.895Zcreating /scratch/u2_3.vdev
1782026-01-17T15:09:50.895Zcreating /scratch/u2_4.vdev
1792026-01-17T15:09:50.895Zcreating /scratch/u2_5.vdev
1802026-01-17T15:09:50.895Zcreating /scratch/u2_6.vdev
1812026-01-17T15:09:50.895Zcreating /scratch/u2_7.vdev
1822026-01-17T15:09:50.895Zcreating /scratch/u2_8.vdev
1832026-01-17T15:09:50.918ZSimnet net0/sc0_0 exists
1842026-01-17T15:09:50.918ZSimnet net1/sc1_0 exists
1852026-01-17T15:09:50.942ZVnic sc0_1 exists
1862026-01-17T15:10:03.187ZUsing 10.151.2.100 as gateway ip
1872026-01-17T15:10:03.187Zusing 18:C0:4D:0D:A0:2A as gateway mac
1882026-01-17T15:10:03.187Zconfiguring SoftNPU ARP entry
1892026-01-17T15:10:03.215Zconfiguring SoftNPU proxy ARP
1902026-01-17T15:10:03.266ZSoftNPU state:
1912026-01-17T15:10:03.266Z local v6:
1922026-01-17T15:10:03.266Z local v4:
1932026-01-17T15:10:03.266Z router v6_idx:
1942026-01-17T15:10:03.266Z router v6_routes:
1952026-01-17T15:10:03.266Z router v4_idx:
1962026-01-17T15:10:03.266Z router v4_routes:
1972026-01-17T15:10:03.266Z resolver v4:
1982026-01-17T15:10:03.266Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
1992026-01-17T15:10:03.266Z resolver v6:
2002026-01-17T15:10:03.266Z nat v4:
2012026-01-17T15:10:03.266Z nat_v6:
2022026-01-17T15:10:03.266Z port_mac:
2032026-01-17T15:10:03.266Z proxy arp:
2042026-01-17T15:10:03.266Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2052026-01-17T15:10:03.266Zcreated virtual hardware
2062026-01-17T15:10:03.290Z
2072026-01-17T15:10:03.290Zreal 12.466394560
2082026-01-17T15:10:03.290Zuser 0.619201309
2092026-01-17T15:10:03.290Zsys 3.105874359
2102026-01-17T15:10:03.290Ztrap 0.000070936
2112026-01-17T15:10:03.290Ztflt 0.000153648
2122026-01-17T15:10:03.290Zdflt 0.001205310
2132026-01-17T15:10:03.290Zkflt 0.000003630
2142026-01-17T15:10:03.290Zlock 9:42.519003736
2152026-01-17T15:10:03.291Zslp 37.627159041
2162026-01-17T15:10:03.291Zlat 0.098447999
2172026-01-17T15:10:03.291Zstop 0.389423042
2182026-01-17T15:10:03.291Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2192026-01-17T15:10:03.326Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2202026-01-17T15:10:03.375Z+ diff -u pkg/config.toml~ pkg/config.toml
2212026-01-17T15:10:03.375Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2222026-01-17T15:10:03.375Z+++ pkg/config.toml Sat Jan 17 15:10:03 2026
2232026-01-17T15:10:03.375Z@@ -29,18 +29,18 @@
2242026-01-17T15:10:03.376Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2252026-01-17T15:10:03.376Z # followed by a string that is embedded into their fake serial values.
2262026-01-17T15:10:03.376Z vdevs = [
2272026-01-17T15:10:03.376Z- "m2_0.vdev",
2282026-01-17T15:10:03.376Z- "m2_1.vdev",
2292026-01-17T15:10:03.376Z+ "/scratch/m2_0.vdev",
2302026-01-17T15:10:03.376Z+ "/scratch/m2_1.vdev",
2312026-01-17T15:10:03.376Z
2322026-01-17T15:10:03.376Z- "u2_0.vdev",
2332026-01-17T15:10:03.376Z- "u2_1.vdev",
2342026-01-17T15:10:03.376Z- "u2_2.vdev",
2352026-01-17T15:10:03.376Z- "u2_3.vdev",
2362026-01-17T15:10:03.376Z- "u2_4.vdev",
2372026-01-17T15:10:03.376Z- "u2_5.vdev",
2382026-01-17T15:10:03.376Z- "u2_6.vdev",
2392026-01-17T15:10:03.376Z- "u2_7.vdev",
2402026-01-17T15:10:03.376Z- "u2_8.vdev",
2412026-01-17T15:10:03.376Z+ "/scratch/u2_0.vdev",
2422026-01-17T15:10:03.376Z+ "/scratch/u2_1.vdev",
2432026-01-17T15:10:03.376Z+ "/scratch/u2_2.vdev",
2442026-01-17T15:10:03.376Z+ "/scratch/u2_3.vdev",
2452026-01-17T15:10:03.376Z+ "/scratch/u2_4.vdev",
2462026-01-17T15:10:03.376Z+ "/scratch/u2_5.vdev",
2472026-01-17T15:10:03.376Z+ "/scratch/u2_6.vdev",
2482026-01-17T15:10:03.376Z+ "/scratch/u2_7.vdev",
2492026-01-17T15:10:03.376Z+ "/scratch/u2_8.vdev",
2502026-01-17T15:10:03.376Z ]
2512026-01-17T15:10:03.376Z
2522026-01-17T15:10:03.376Z # The amount of memory held back for services which exist between zero and one
2532026-01-17T15:10:03.376Z+ true
2542026-01-17T15:10:03.376Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2552026-01-17T15:10:03.376Z+ EXPECTED_ZPOOL_COUNT=9
2562026-01-17T15:10:03.377Z+ echo 'expected number of zpools is 9'
2572026-01-17T15:10:03.377Zexpected number of zpools is 9
2582026-01-17T15:10:03.377Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2592026-01-17T15:10:03.377Z+ SILO_NAME=recovery
2602026-01-17T15:10:03.377Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2612026-01-17T15:10:03.377Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2622026-01-17T15:10:03.377Z+ sed -i~ '
2632026-01-17T15:10:03.377Z /^external_dns_ips/c\
2642026-01-17T15:10:03.377Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2652026-01-17T15:10:03.377Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2662026-01-17T15:10:03.377Z /^first/c\
2672026-01-17T15:10:03.377Zfirst = "10.151.2.164"
2682026-01-17T15:10:03.377Z /^last/c\
2692026-01-17T15:10:03.377Zlast = "10.151.2.173"
2702026-01-17T15:10:03.377Z }
2712026-01-17T15:10:03.377Z /^infra_ip_first/c\
2722026-01-17T15:10:03.377Zinfra_ip_first = "10.151.2.174"
2732026-01-17T15:10:03.377Z /^infra_ip_last/c\
2742026-01-17T15:10:03.377Zinfra_ip_last = "10.151.2.174"
2752026-01-17T15:10:03.377Z /^\[\[rack_network_config.ports/,/^$/ {
2762026-01-17T15:10:03.377Z /^routes/c\
2772026-01-17T15:10:03.377Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2782026-01-17T15:10:03.377Z /^addresses/c\
2792026-01-17T15:10:03.377Zaddresses = \[{address = "10.151.2.174/24"} \]
2802026-01-17T15:10:03.377Z }
2812026-01-17T15:10:03.377Z' pkg/config-rss.toml
2822026-01-17T15:10:03.377Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2832026-01-17T15:10:03.377Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2842026-01-17T15:10:03.377Z+++ pkg/config-rss.toml Sat Jan 17 15:10:03 2026
2852026-01-17T15:10:03.377Z@@ -33,7 +33,7 @@
2862026-01-17T15:10:03.377Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2872026-01-17T15:10:03.377Z # must be contained in one of the "internal services" IP Pool ranges listed
2882026-01-17T15:10:03.377Z # below.
2892026-01-17T15:10:03.377Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2902026-01-17T15:10:03.377Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2912026-01-17T15:10:03.377Z
2922026-01-17T15:10:03.377Z # Initial TLS certificates for the external API
2932026-01-17T15:10:03.377Z #
2942026-01-17T15:10:03.378Z@@ -69,8 +69,8 @@
2952026-01-17T15:10:03.378Z #
2962026-01-17T15:10:03.378Z # For more on this and what to put here, see docs/how-to-run.adoc.
2972026-01-17T15:10:03.378Z [[internal_services_ip_pool_ranges]]
2982026-01-17T15:10:03.378Z-first = "192.168.1.20"
2992026-01-17T15:10:03.378Z-last = "192.168.1.29"
3002026-01-17T15:10:03.378Z+first = "10.151.2.164"
3012026-01-17T15:10:03.378Z+last = "10.151.2.173"
3022026-01-17T15:10:03.378Z
3032026-01-17T15:10:03.378Z # TODO - this configuration is subject to change going forward. Ultimately these
3042026-01-17T15:10:03.378Z # parameters should be provided to the control plane via wicket, but we need to
3052026-01-17T15:10:03.378Z@@ -91,8 +91,8 @@
3062026-01-17T15:10:03.378Z # A range of IP addresses used by Boundary Services on the external network. In
3072026-01-17T15:10:03.378Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3082026-01-17T15:10:03.378Z # With softnpu, only one address is used.
3092026-01-17T15:10:03.378Z-infra_ip_first = "192.168.1.30"
3102026-01-17T15:10:03.378Z-infra_ip_last = "192.168.1.30"
3112026-01-17T15:10:03.378Z+infra_ip_first = "10.151.2.174"
3122026-01-17T15:10:03.378Z+infra_ip_last = "10.151.2.174"
3132026-01-17T15:10:03.378Z
3142026-01-17T15:10:03.378Z # Configurations for BGP routers to run on the scrimlets.
3152026-01-17T15:10:03.378Z bgp = []
3162026-01-17T15:10:03.379Z@@ -100,9 +100,9 @@
3172026-01-17T15:10:03.379Z # You can configure multiple uplinks by repeating the following stanza
3182026-01-17T15:10:03.379Z [[rack_network_config.ports]]
3192026-01-17T15:10:03.379Z # Routes associated with this port.
3202026-01-17T15:10:03.403Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3212026-01-17T15:10:03.403Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3222026-01-17T15:10:03.403Z # Addresses associated with this port.
3232026-01-17T15:10:03.403Z-addresses = [{address = "192.168.1.30/24"}]
3242026-01-17T15:10:03.403Z+addresses = [{address = "10.151.2.174/24"} ]
3252026-01-17T15:10:03.403Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3262026-01-17T15:10:03.403Z port = "qsfp0"
3272026-01-17T15:10:03.403Z # The speed of this port.
3282026-01-17T15:10:03.403Z+ true
3292026-01-17T15:10:03.403Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3302026-01-17T15:10:03.403Za pkg/config-rss.toml 8K
3312026-01-17T15:10:03.403Za pkg/config.toml 6K
3322026-01-17T15:10:03.403Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3332026-01-17T15:10:03.403Z+ TLS_NAME=recovery.sys.oxide.test
3342026-01-17T15:10:03.403Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3352026-01-17T15:10:03.839Z.............+.+.....+...+....+...+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*......+.+......+........+.........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*......+..+......+.+..............+.......+..+......+........................+.......+.........+............+........+..........+....................+......+......+.........................+............+..+.............+........+...+....+...+..+.+...........+......+....+.....+..........+.........+..+............+.+..+.......+........+.......+........+............+.+......+.....+.+..+.+..+.......+...........+......+....+...........+..........+...+.....+...............+...+....+.......................+.+..+.......+...........+......................+............+......+..+....+.........+.........+.....+......+.......+..+.+.....+....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3362026-01-17T15:10:03.977Z.+.+..............+......+...+.+.....+...+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+....+...+..+.+..+.............+.....+....+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...............+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3372026-01-17T15:10:04.000Z-----
3382026-01-17T15:10:04.024Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3392026-01-17T15:10:04.024Za pkg/initial-tls-cert.pem 2K
3402026-01-17T15:10:04.024Za pkg/initial-tls-key.pem 4K
3412026-01-17T15:10:04.024Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3422026-01-17T15:10:04.024Z+ rmdir pkg
3432026-01-17T15:10:04.024Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3442026-01-17T15:10:04.024Z+ pfexec mkdir -p /usr/oxide
3452026-01-17T15:10:04.024Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3462026-01-17T15:10:04.686Z+ pfexec chmod +x /usr/oxide/oxide
3472026-01-17T15:10:04.709Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3482026-01-17T15:10:23.469Z+ OMICRON_NO_UNINSTALL=1
3492026-01-17T15:10:23.469Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3502026-01-17T15:10:23.493ZLogging to: /opt/oxide/work/out/LOG
3512026-01-17T15:10:24.305Z
3522026-01-17T15:10:24.329Zreal 0.830691251
3532026-01-17T15:10:24.329Zuser 0.177384948
3542026-01-17T15:10:24.329Zsys 4.798755281
3552026-01-17T15:10:24.329Ztrap 0.000002180
3562026-01-17T15:10:24.329Ztflt 0.000025229
3572026-01-17T15:10:24.329Zdflt 0.000111164
3582026-01-17T15:10:24.329Zkflt 0.000000000
3592026-01-17T15:10:24.329Zlock 1:14.854030592
3602026-01-17T15:10:24.329Zslp 1.114597309
3612026-01-17T15:10:24.329Zlat 0.033113674
3622026-01-17T15:10:24.329Zstop 0.053962388
3632026-01-17T15:10:24.329Z+ retry=0
3642026-01-17T15:10:24.329Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3652026-01-17T15:10:24.329Z+ [[ 0 -gt 30 ]]
3662026-01-17T15:10:24.329Z+ sleep 1
3672026-01-17T15:10:25.320Z+ retry=1
3682026-01-17T15:10:25.320Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3692026-01-17T15:10:25.343Z+ [[ 1 -gt 30 ]]
3702026-01-17T15:10:25.343Z+ sleep 1
3712026-01-17T15:10:26.334Z+ retry=2
3722026-01-17T15:10:26.334Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3732026-01-17T15:10:26.358Z+ [[ 2 -gt 30 ]]
3742026-01-17T15:10:26.358Z+ sleep 1
3752026-01-17T15:10:27.351Z+ retry=3
3762026-01-17T15:10:27.351Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3772026-01-17T15:10:47.267Z+ [[ 3 -gt 30 ]]
3782026-01-17T15:10:47.267Z+ sleep 1
3792026-01-17T15:10:48.270Z+ retry=4
3802026-01-17T15:10:48.270Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3812026-01-17T15:10:48.294Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
3822026-01-17T15:10:48.317Zlocal v6:
3832026-01-17T15:10:48.317Zfe80::7cb9:b4ff:fec4:a117
3842026-01-17T15:10:48.317Zfd41:8e27:cfde:7db8::1
3852026-01-17T15:10:48.317Zlocal v4:
3862026-01-17T15:10:48.317Zrouter v6_idx:
3872026-01-17T15:10:48.317Zrouter v6_routes:
3882026-01-17T15:10:48.317Zrouter v4_idx:
3892026-01-17T15:10:48.317Zrouter v4_routes:
3902026-01-17T15:10:48.317Zresolver v4:
3912026-01-17T15:10:48.317Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
3922026-01-17T15:10:48.318Zresolver v6:
3932026-01-17T15:10:48.318Zfe80::7cb9:b4ff:fec4:a117 -> 7e:b9:b4:c4:a1:17
3942026-01-17T15:10:48.318Zfe80::606d:edff:fe29:b10a -> 62:6d:ed:29:b1:0a
3952026-01-17T15:10:48.318Znat v4:
3962026-01-17T15:10:48.318Znat_v6:
3972026-01-17T15:10:48.318Zport_mac:
3982026-01-17T15:10:48.318Z1000: a8:40:25:72:39:c3
3992026-01-17T15:10:48.318Z1: a8:40:25:72:39:c4
4002026-01-17T15:10:48.318Zproxy arp:
4012026-01-17T15:10:48.318Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
4022026-01-17T15:10:48.318Z+ retry=0
4032026-01-17T15:10:48.342Z+++ zoneadm list -n
4042026-01-17T15:10:48.342Z+++ grep oxz_ntp
4052026-01-17T15:10:48.342Z++ pfexec svcs -z -Hostate oxide/ntp
4062026-01-17T15:10:48.343Zsvcs: zone '-Hostate': entity not found
4072026-01-17T15:10:48.343Z++ true
4082026-01-17T15:10:48.343Z+ [[ '' != online ]]
4092026-01-17T15:10:48.343Z+ [[ 0 -gt 60 ]]
4102026-01-17T15:10:48.343Z+ sleep 1
4112026-01-17T15:10:49.343Z+ retry=1
4122026-01-17T15:10:49.368Z+++ zoneadm list -n
4132026-01-17T15:10:49.368Z+++ grep oxz_ntp
4142026-01-17T15:10:49.368Z++ pfexec svcs -z -Hostate oxide/ntp
4152026-01-17T15:10:49.368Zsvcs: zone '-Hostate': entity not found
4162026-01-17T15:10:49.368Z++ true
4172026-01-17T15:10:49.368Z+ [[ '' != online ]]
4182026-01-17T15:10:49.368Z+ [[ 1 -gt 60 ]]
4192026-01-17T15:10:49.368Z+ sleep 1
4202026-01-17T15:10:50.366Z+ retry=2
4212026-01-17T15:10:50.391Z+++ zoneadm list -n
4222026-01-17T15:10:50.392Z+++ grep oxz_ntp
4232026-01-17T15:10:50.392Z++ pfexec svcs -z -Hostate oxide/ntp
4242026-01-17T15:10:50.392Zsvcs: zone '-Hostate': entity not found
4252026-01-17T15:10:50.392Z++ true
4262026-01-17T15:10:50.392Z+ [[ '' != online ]]
4272026-01-17T15:10:50.392Z+ [[ 2 -gt 60 ]]
4282026-01-17T15:10:50.392Z+ sleep 1
4292026-01-17T15:10:51.389Z+ retry=3
4302026-01-17T15:10:51.413Z+++ zoneadm list -n
4312026-01-17T15:10:51.413Z+++ grep oxz_ntp
4322026-01-17T15:10:51.413Z++ pfexec svcs -z -Hostate oxide/ntp
4332026-01-17T15:10:51.413Zsvcs: zone '-Hostate': entity not found
4342026-01-17T15:10:51.413Z++ true
4352026-01-17T15:10:51.413Z+ [[ '' != online ]]
4362026-01-17T15:10:51.413Z+ [[ 3 -gt 60 ]]
4372026-01-17T15:10:51.413Z+ sleep 1
4382026-01-17T15:10:52.411Z+ retry=4
4392026-01-17T15:10:52.434Z+++ zoneadm list -n
4402026-01-17T15:10:52.434Z+++ grep oxz_ntp
4412026-01-17T15:10:52.434Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4422026-01-17T15:10:52.465Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4432026-01-17T15:10:52.488Z++ true
4442026-01-17T15:10:52.489Z+ [[ '' != online ]]
4452026-01-17T15:10:52.489Z+ [[ 4 -gt 60 ]]
4462026-01-17T15:10:52.489Z+ sleep 1
4472026-01-17T15:10:53.471Z+ retry=5
4482026-01-17T15:10:53.494Z+++ zoneadm list -n
4492026-01-17T15:10:53.495Z+++ grep oxz_ntp
4502026-01-17T15:10:53.495Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4512026-01-17T15:10:53.523Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4522026-01-17T15:10:53.523Z++ true
4532026-01-17T15:10:53.523Z+ [[ '' != online ]]
4542026-01-17T15:10:53.523Z+ [[ 5 -gt 60 ]]
4552026-01-17T15:10:53.523Z+ sleep 1
4562026-01-17T15:10:54.501Z+ retry=6
4572026-01-17T15:10:54.525Z+++ zoneadm list -n
4582026-01-17T15:10:54.525Z+++ grep oxz_ntp
4592026-01-17T15:10:54.525Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4602026-01-17T15:10:54.525Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4612026-01-17T15:10:54.549Z++ true
4622026-01-17T15:10:54.549Z+ [[ '' != online ]]
4632026-01-17T15:10:54.549Z+ [[ 6 -gt 60 ]]
4642026-01-17T15:10:54.549Z+ sleep 1
4652026-01-17T15:10:55.530Z+ retry=7
4662026-01-17T15:10:55.554Z+++ zoneadm list -n
4672026-01-17T15:10:55.554Z+++ grep oxz_ntp
4682026-01-17T15:10:55.554Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4692026-01-17T15:10:55.577Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4702026-01-17T15:10:55.577Z++ true
4712026-01-17T15:10:55.577Z+ [[ '' != online ]]
4722026-01-17T15:10:55.577Z+ [[ 7 -gt 60 ]]
4732026-01-17T15:10:55.577Z+ sleep 1
4742026-01-17T15:10:56.559Z+ retry=8
4752026-01-17T15:10:56.582Z+++ zoneadm list -n
4762026-01-17T15:10:56.582Z+++ grep oxz_ntp
4772026-01-17T15:10:56.582Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4782026-01-17T15:10:56.582Z+ [[ offline != online ]]
4792026-01-17T15:10:56.582Z+ [[ 8 -gt 60 ]]
4802026-01-17T15:10:56.582Z+ sleep 1
4812026-01-17T15:10:57.582Z+ retry=9
4822026-01-17T15:10:57.606Z+++ zoneadm list -n
4832026-01-17T15:10:57.606Z+++ grep oxz_ntp
4842026-01-17T15:10:57.606Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4852026-01-17T15:10:57.606Z+ [[ offline != online ]]
4862026-01-17T15:10:57.606Z+ [[ 9 -gt 60 ]]
4872026-01-17T15:10:57.606Z+ sleep 1
4882026-01-17T15:10:58.605Z+ retry=10
4892026-01-17T15:10:58.629Z+++ zoneadm list -n
4902026-01-17T15:10:58.629Z+++ grep oxz_ntp
4912026-01-17T15:10:58.629Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4922026-01-17T15:10:58.629Z+ [[ offline != online ]]
4932026-01-17T15:10:58.629Z+ [[ 10 -gt 60 ]]
4942026-01-17T15:10:58.629Z+ sleep 1
4952026-01-17T15:10:59.627Z+ retry=11
4962026-01-17T15:10:59.651Z+++ zoneadm list -n
4972026-01-17T15:10:59.651Z+++ grep oxz_ntp
4982026-01-17T15:10:59.651Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
4992026-01-17T15:10:59.651Z+ [[ offline != online ]]
5002026-01-17T15:10:59.651Z+ [[ 11 -gt 60 ]]
5012026-01-17T15:10:59.651Z+ sleep 1
5022026-01-17T15:11:00.650Z+ retry=12
5032026-01-17T15:11:00.674Z+++ zoneadm list -n
5042026-01-17T15:11:00.674Z+++ grep oxz_ntp
5052026-01-17T15:11:00.674Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
5062026-01-17T15:11:00.674Z+ [[ offline != online ]]
5072026-01-17T15:11:00.674Z+ [[ 12 -gt 60 ]]
5082026-01-17T15:11:00.674Z+ sleep 1
5092026-01-17T15:11:01.672Z+ retry=13
5102026-01-17T15:11:01.696Z+++ zoneadm list -n
5112026-01-17T15:11:01.696Z+++ grep oxz_ntp
5122026-01-17T15:11:01.696Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
5132026-01-17T15:11:01.696Z+ [[ offline != online ]]
5142026-01-17T15:11:01.696Z+ [[ 13 -gt 60 ]]
5152026-01-17T15:11:01.696Z+ sleep 1
5162026-01-17T15:11:02.693Z+ retry=14
5172026-01-17T15:11:02.716Z+++ zoneadm list -n
5182026-01-17T15:11:02.717Z+++ grep oxz_ntp
5192026-01-17T15:11:02.717Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
5202026-01-17T15:11:02.717Z+ [[ offline != online ]]
5212026-01-17T15:11:02.717Z+ [[ 14 -gt 60 ]]
5222026-01-17T15:11:02.717Z+ sleep 1
5232026-01-17T15:11:03.714Z+ retry=15
5242026-01-17T15:11:03.738Z+++ zoneadm list -n
5252026-01-17T15:11:03.738Z+++ grep oxz_ntp
5262026-01-17T15:11:03.738Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
5272026-01-17T15:11:03.738Z+ [[ offline != online ]]
5282026-01-17T15:11:03.738Z+ [[ 15 -gt 60 ]]
5292026-01-17T15:11:03.738Z+ sleep 1
 
5302026-01-17T15:11:03.823Zsock console login: Jan 17 15:11:03 sock ip: ip: joining multicasts failed (4) on opte0 - will use link layer broadcasts for multicast
 
5312026-01-17T15:11:04.737Z+ retry=16
5322026-01-17T15:11:04.761Z+++ zoneadm list -n
5332026-01-17T15:11:04.761Z+++ grep oxz_ntp
5342026-01-17T15:11:04.761Z++ pfexec svcs -z oxz_ntp_4fe32ac3-6c49-46b7-9e0b-b1b2026ed786 -Hostate oxide/ntp
5352026-01-17T15:11:04.761Z+ [[ online != online ]]
5362026-01-17T15:11:04.761Z+ echo 'Waited for chrony: 16s'
5372026-01-17T15:11:04.761Z+ retry=0
5382026-01-17T15:11:04.761ZWaited for chrony: 16s
5392026-01-17T15:11:04.761Z+ zoneadm list
5402026-01-17T15:11:04.761Z+ grep nexus
5412026-01-17T15:11:04.784Z+ [[ 0 -gt 300 ]]
5422026-01-17T15:11:04.784Z+ sleep 1
5432026-01-17T15:11:05.767Z+ retry=1
5442026-01-17T15:11:05.790Z+ zoneadm list
5452026-01-17T15:11:05.790Z+ grep nexus
5462026-01-17T15:11:05.790Z+ [[ 1 -gt 300 ]]
5472026-01-17T15:11:05.790Z+ sleep 1
5482026-01-17T15:11:06.780Z+ retry=2
5492026-01-17T15:11:06.803Z+ zoneadm list
5502026-01-17T15:11:06.803Z+ grep nexus
5512026-01-17T15:11:06.803Z+ [[ 2 -gt 300 ]]
5522026-01-17T15:11:06.803Z+ sleep 1
5532026-01-17T15:11:07.793Z+ retry=3
5542026-01-17T15:11:07.816Z+ zoneadm list
5552026-01-17T15:11:07.817Z+ grep nexus
5562026-01-17T15:11:07.817Z+ [[ 3 -gt 300 ]]
5572026-01-17T15:11:07.817Z+ sleep 1
5582026-01-17T15:11:08.806Z+ retry=4
5592026-01-17T15:11:08.830Z+ zoneadm list
5602026-01-17T15:11:08.830Z+ grep nexus
5612026-01-17T15:11:08.830Z+ [[ 4 -gt 300 ]]
5622026-01-17T15:11:08.830Z+ sleep 1
5632026-01-17T15:11:09.820Z+ retry=5
5642026-01-17T15:11:09.843Z+ zoneadm list
5652026-01-17T15:11:09.843Z+ grep nexus
5662026-01-17T15:11:09.843Z+ [[ 5 -gt 300 ]]
5672026-01-17T15:11:09.843Z+ sleep 1
5682026-01-17T15:11:10.833Z+ retry=6
5692026-01-17T15:11:10.857Z+ zoneadm list
5702026-01-17T15:11:10.857Z+ grep nexus
5712026-01-17T15:11:10.857Z+ [[ 6 -gt 300 ]]
5722026-01-17T15:11:10.857Z+ sleep 1
5732026-01-17T15:11:11.847Z+ retry=7
5742026-01-17T15:11:11.871Z+ zoneadm list
5752026-01-17T15:11:11.871Z+ grep nexus
5762026-01-17T15:11:11.871Z+ [[ 7 -gt 300 ]]
5772026-01-17T15:11:11.871Z+ sleep 1
5782026-01-17T15:11:12.859Z+ retry=8
5792026-01-17T15:11:12.883Z+ zoneadm list
5802026-01-17T15:11:12.884Z+ grep nexus
5812026-01-17T15:11:12.884Z+ [[ 8 -gt 300 ]]
5822026-01-17T15:11:12.884Z+ sleep 1
5832026-01-17T15:11:13.872Z+ retry=9
5842026-01-17T15:11:13.896Z+ zoneadm list
5852026-01-17T15:11:13.896Z+ grep nexus
5862026-01-17T15:11:13.896Z+ [[ 9 -gt 300 ]]
5872026-01-17T15:11:13.896Z+ sleep 1
5882026-01-17T15:11:14.886Z+ retry=10
5892026-01-17T15:11:14.910Z+ zoneadm list
5902026-01-17T15:11:14.910Z+ grep nexus
5912026-01-17T15:11:14.910Z+ [[ 10 -gt 300 ]]
5922026-01-17T15:11:14.910Z+ sleep 1
5932026-01-17T15:11:15.900Z+ retry=11
5942026-01-17T15:11:15.924Z+ zoneadm list
5952026-01-17T15:11:15.924Z+ grep nexus
5962026-01-17T15:11:15.924Z+ [[ 11 -gt 300 ]]
5972026-01-17T15:11:15.924Z+ sleep 1
5982026-01-17T15:11:16.913Z+ retry=12
5992026-01-17T15:11:16.937Z+ zoneadm list
6002026-01-17T15:11:16.937Z+ grep nexus
6012026-01-17T15:11:16.937Z+ [[ 12 -gt 300 ]]
6022026-01-17T15:11:16.937Z+ sleep 1
6032026-01-17T15:11:17.926Z+ retry=13
6042026-01-17T15:11:17.950Z+ zoneadm list
6052026-01-17T15:11:17.950Z+ grep nexus
6062026-01-17T15:11:17.950Z+ [[ 13 -gt 300 ]]
6072026-01-17T15:11:17.950Z+ sleep 1
6082026-01-17T15:11:18.939Z+ retry=14
6092026-01-17T15:11:18.963Z+ zoneadm list
6102026-01-17T15:11:18.963Z+ grep nexus
6112026-01-17T15:11:18.963Z+ [[ 14 -gt 300 ]]
6122026-01-17T15:11:18.963Z+ sleep 1
6132026-01-17T15:11:19.952Z+ retry=15
6142026-01-17T15:11:19.976Z+ zoneadm list
6152026-01-17T15:11:19.976Z+ grep nexus
6162026-01-17T15:11:19.977Z+ [[ 15 -gt 300 ]]
6172026-01-17T15:11:19.977Z+ sleep 1
6182026-01-17T15:11:20.966Z+ retry=16
6192026-01-17T15:11:20.990Z+ zoneadm list
6202026-01-17T15:11:20.990Z+ grep nexus
6212026-01-17T15:11:20.990Z+ [[ 16 -gt 300 ]]
6222026-01-17T15:11:20.990Z+ sleep 1
6232026-01-17T15:11:21.979Z+ retry=17
6242026-01-17T15:11:22.003Z+ zoneadm list
6252026-01-17T15:11:22.003Z+ grep nexus
6262026-01-17T15:11:22.003Z+ [[ 17 -gt 300 ]]
6272026-01-17T15:11:22.003Z+ sleep 1
6282026-01-17T15:11:22.994Z+ retry=18
6292026-01-17T15:11:23.017Z+ zoneadm list
6302026-01-17T15:11:23.017Z+ grep nexus
6312026-01-17T15:11:23.017Z+ [[ 18 -gt 300 ]]
6322026-01-17T15:11:23.017Z+ sleep 1
6332026-01-17T15:11:24.008Z+ retry=19
6342026-01-17T15:11:24.031Z+ zoneadm list
6352026-01-17T15:11:24.032Z+ grep nexus
6362026-01-17T15:11:24.032Z+ [[ 19 -gt 300 ]]
6372026-01-17T15:11:24.032Z+ sleep 1
6382026-01-17T15:11:25.022Z+ retry=20
6392026-01-17T15:11:25.046Z+ zoneadm list
6402026-01-17T15:11:25.046Z+ grep nexus
6412026-01-17T15:11:25.046Z+ [[ 20 -gt 300 ]]
6422026-01-17T15:11:25.046Z+ sleep 1
6432026-01-17T15:11:26.040Z+ retry=21
6442026-01-17T15:11:26.064Z+ zoneadm list
6452026-01-17T15:11:26.064Z+ grep nexus
6462026-01-17T15:11:26.064Z+ [[ 21 -gt 300 ]]
6472026-01-17T15:11:26.064Z+ sleep 1
6482026-01-17T15:11:27.059Z+ retry=22
6492026-01-17T15:11:27.082Z+ zoneadm list
6502026-01-17T15:11:27.082Z+ grep nexus
6512026-01-17T15:11:27.082Z+ [[ 22 -gt 300 ]]
6522026-01-17T15:11:27.082Z+ sleep 1
6532026-01-17T15:11:28.078Z+ retry=23
6542026-01-17T15:11:28.102Z+ zoneadm list
6552026-01-17T15:11:28.102Z+ grep nexus
6562026-01-17T15:11:28.102Z+ [[ 23 -gt 300 ]]
6572026-01-17T15:11:28.102Z+ sleep 1
6582026-01-17T15:11:29.102Z+ retry=24
6592026-01-17T15:11:29.126Z+ zoneadm list
6602026-01-17T15:11:29.126Z+ grep nexus
6612026-01-17T15:11:29.126Z+ [[ 24 -gt 300 ]]
6622026-01-17T15:11:29.126Z+ sleep 1
6632026-01-17T15:11:30.115Z+ retry=25
6642026-01-17T15:11:30.138Z+ zoneadm list
6652026-01-17T15:11:30.139Z+ grep nexus
6662026-01-17T15:11:30.139Z+ [[ 25 -gt 300 ]]
6672026-01-17T15:11:30.139Z+ sleep 1
6682026-01-17T15:11:31.132Z+ retry=26
6692026-01-17T15:11:31.155Z+ zoneadm list
6702026-01-17T15:11:31.155Z+ grep nexus
6712026-01-17T15:11:31.155Z+ [[ 26 -gt 300 ]]
6722026-01-17T15:11:31.155Z+ sleep 1
6732026-01-17T15:11:32.151Z+ retry=27
6742026-01-17T15:11:32.174Z+ zoneadm list
6752026-01-17T15:11:32.174Z+ grep nexus
6762026-01-17T15:11:32.174Z+ [[ 27 -gt 300 ]]
6772026-01-17T15:11:32.174Z+ sleep 1
6782026-01-17T15:11:33.165Z+ retry=28
6792026-01-17T15:11:33.189Z+ zoneadm list
6802026-01-17T15:11:33.189Z+ grep nexus
6812026-01-17T15:11:33.189Z+ [[ 28 -gt 300 ]]
6822026-01-17T15:11:33.189Z+ sleep 1
6832026-01-17T15:11:34.180Z+ retry=29
6842026-01-17T15:11:34.204Z+ zoneadm list
6852026-01-17T15:11:34.204Z+ grep nexus
6862026-01-17T15:11:34.204Z+ [[ 29 -gt 300 ]]
6872026-01-17T15:11:34.205Z+ sleep 1
6882026-01-17T15:11:35.197Z+ retry=30
6892026-01-17T15:11:35.220Z+ zoneadm list
6902026-01-17T15:11:35.221Z+ grep nexus
6912026-01-17T15:11:35.221Z+ [[ 30 -gt 300 ]]
6922026-01-17T15:11:35.221Z+ sleep 1
6932026-01-17T15:11:36.212Z+ retry=31
6942026-01-17T15:11:36.236Z+ zoneadm list
6952026-01-17T15:11:36.236Z+ grep nexus
6962026-01-17T15:11:36.236Z+ [[ 31 -gt 300 ]]
6972026-01-17T15:11:36.236Z+ sleep 1
 
6982026-01-17T15:11:40.033ZWARNING: panicked at lib/opte/src/ddi/mblk.rs:290:14:
6992026-01-17T15:11:40.082Zsomehow failed to get an mblk...
7002026-01-17T15:11:40.131Zpanic[cpu11]/thread=fffffe85e6c33820: panicked at lib/opte/src/ddi/mblk.rs:290:14:
7012026-01-17T15:11:40.177Zsomehow failed to get an mblk...
7022026-01-17T15:11:40.216Zfffffe00c50e4d90 xde:_RNvCsj0qrnF3AvHz_7___rustc17rust_begin_unwind+16d ()
7032026-01-17T15:11:40.261Zfffffe00c50e4dc0 xde:opte_panic_debug+33fe576b ()
7042026-01-17T15:11:40.306Zfffffe00c50e4e20 xde:opte_panic_debug+33fe5c20 ()
7052026-01-17T15:11:40.357Zfffffe00c50e4ed0 xde:_ZN4opte6engine6packet8EmitSpec5apply17hcdb7dfa0174ed03bE+e21 ()
7062026-01-17T15:11:40.402Zfffffe00c50e53e0 xde:xde_mc_tx+3b88 ()
7072026-01-17T15:11:40.447Zfffffe00c50e5410 mac:mac_ring_tx+50 ()
7082026-01-17T15:11:40.492Zfffffe00c50e5460 mac:mac_provider_tx+6d ()
7092026-01-17T15:11:40.538Zfffffe00c50e5520 mac:mac_tx+28b ()
7102026-01-17T15:11:40.594Zfffffe00c50e5570 dld:str_mdata_fastpath_put+8e ()
7112026-01-17T15:11:40.639Zfffffe00c50e5680 ip:ip_xmit+9ce ()
7122026-01-17T15:11:40.717Zfffffe00c50e58d0 ip:ire_send_wire_v4+3c2 ()
7132026-01-17T15:11:40.762Zfffffe00c50e5950 ip:conn_ip_output+18f ()
7142026-01-17T15:11:40.813Zfffffe00c50e5a30 ip:udp_output_connected+131 ()
7152026-01-17T15:11:40.858Zfffffe00c50e5ad0 ip:udp_send+1bf ()
7162026-01-17T15:11:40.909Zfffffe00c50e5b60 sockfs:so_sendmsg+201 ()
7172026-01-17T15:11:40.954Zfffffe00c50e5bc0 sockfs:socket_sendmsg+65 ()
7182026-01-17T15:11:40.988Zfffffe00c50e5c60 sockfs:sendit+186 ()
7192026-01-17T15:11:41.037Zfffffe00c50e5f00 sockfs:sendmsg+2c6 ()
7202026-01-17T15:11:41.071Zfffffe00c50e5f10 unix:brand_sys_syscall+1fe ()
7212026-01-17T15:11:41.104Zskipping system dump - no dump device configured
7222026-01-17T15:11:41.138Zrebooting...
7232026-01-17T15:12:28.165Z^[[2J^[[1;1H^[[8;25;80t^[[2J^[[1;1H^[[1;1H^[[0;37;40m^[[2J^[[1;1H^[[25;
7242026-01-17T15:12:28.493Z8H98^[[1;1H^[[25;78H9D^[[1;1H^[[25;78H9A^[[1;1H^[[25;78H9C^[[1;1H^[[25;78
7252026-01-17T15:12:28.540Z9D^[[1;1H^[[25;78H9A^[[1;1H^[[25;78H9C^[[1;1H^[[25;78HB4^[[1;1H^[[25;78HB
7262026-01-17T15:12:31.602Z^[[1;1H^[[25;78HB4^[[1;1H^[[1;1H^[[1;1HPress <DEL> to enter setup, <F1
7272026-01-17T15:12:32.526Z> Display Boot Menu, <F12> Force Network Boot ^[[25;78H92^[[2;1H^[
7282026-01-17T15:12:32.564Z25;78HA0^[[2;1H^[[25;78HA0^[[2;1H^[[25;78H99^[[2;1H^[[25;78H92^[[2;1H^[[2
7292026-01-17T15:12:32.609Z;78H92^[[2;1H^[[25;78H72^[[2;1H^[[25;78HA0^[[2;1H^[[25;78HA2^[[2;1H^[[25;
7302026-01-17T15:12:32.948Z8HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78
7312026-01-17T15:12:33.307ZA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78H92^[[2;1H^[[25;
7322026-01-17T15:12:33.362Z8HB2^[[2;1H^[[2J^[[1;1H^[[01;00H^[[0m^[[2;37;40mInitializing Intel(R) B
7332026-01-17T15:12:33.418Zot Agent GE v1.5.53 ^[[02;00HP
7342026-01-17T15:12:33.458ZE 2.1 Build 092 (WfM 2.0) ^[
7352026-01-17T15:12:33.503Z03;00HPress Ctrl+S to enter the Setup Menu. ^[
7362026-01-17T15:12:33.548Z04;00H ^[
7372026-01-17T15:12:33.600Z05;00H ^[
7382026-01-17T15:12:33.707Z06;00H ^[
7392026-01-17T15:12:33.757Z07;00H ^[
7402026-01-17T15:12:33.808Z08;00H ^[
7412026-01-17T15:12:33.858Z09;00H ^[
7422026-01-17T15:12:33.909Z10;00H ^[
7432026-01-17T15:12:33.942Z11;00H ^[
7442026-01-17T15:12:33.975Z12;00H ^[
7452026-01-17T15:12:34.009Z13;00H ^[
7462026-01-17T15:12:34.042Z14;00H ^[
7472026-01-17T15:12:34.076Z15;00H ^[
7482026-01-17T15:12:34.148Z16;00H ^[
7492026-01-17T15:12:34.188Z17;00H ^[
7502026-01-17T15:12:34.259Z18;00H ^[
7512026-01-17T15:12:34.304Z19;00H ^[
7522026-01-17T15:12:34.343Z20;00H ^[
7532026-01-17T15:12:34.388Z21;00H ^[
7542026-01-17T15:12:34.433Z22;00H ^[
7552026-01-17T15:12:34.477Z23;00H ^[
7562026-01-17T15:12:34.511Z24;00H ^[
7572026-01-17T15:12:34.544Z24;00H^[[03;39H^[[03;00HPress Ctrl+S to enter the Setup Menu..
7582026-01-17T15:12:35.444Z^[[03;39H^[[01;00HInitializ
7592026-01-17T15:12:35.478Zng Intel(R) Boot Agent GE v1.5.53 ^[
7602026-01-17T15:12:35.511Z02;00HPXE 2.1 Build 092 (WfM 2.0) ^[
7612026-01-17T15:12:35.830Z03;00H ^[
7622026-01-17T15:12:35.867Z04;00H ^[
7632026-01-17T15:12:35.917Z05;00HInitializing Intel(R) Boot Agent GE v1.5.53 ^[
7642026-01-17T15:12:35.963Z06;00HPXE 2.1 Build 092 (WfM 2.0) ^[
7652026-01-17T15:12:36.002Z07;00HPress Ctrl+S to enter the Setup Menu. ^[
7662026-01-17T15:12:36.036Z08;00H ^[
7672026-01-17T15:12:36.069Z09;00H ^[
7682026-01-17T15:12:36.103Z10;00H ^[
7692026-01-17T15:12:36.622Z11;00H ^[
7702026-01-17T15:12:36.690Z12;00H ^[
7712026-01-17T15:12:36.735Z13;00H ^[
7722026-01-17T15:12:36.780Z14;00H ^[
7732026-01-17T15:12:36.825Z15;00H ^[
7742026-01-17T15:12:36.870Z16;00H ^[
7752026-01-17T15:12:36.948Z17;00H ^[
7762026-01-17T15:12:36.992Z18;00H ^[
7772026-01-17T15:12:37.054Z19;00H ^[
7782026-01-17T15:12:37.104Z20;00H ^[
7792026-01-17T15:12:37.137Z21;00H ^[
7802026-01-17T15:12:37.171Z22;00H ^[
7812026-01-17T15:12:37.204Z23;00H ^[
7822026-01-17T15:12:37.237Z24;00H ^[
7832026-01-17T15:12:37.271Z24;00H^[[07;39H^[[07;00HPress Ctrl+S to enter the Setup Menu..
7842026-01-17T15:12:40.184Z^[[07;39H^[[2J^[[1;1H^[[8;25;80t^[
7852026-01-17T15:12:40.222Z2J^[[1;1H^[[1;1H^[[0;37;40mPress <DEL> to enter setup, <F10> Display
7862026-01-17T15:12:40.256ZBoot Menu, <F12> Force Network Boot B2^[
7872026-01-17T15:12:40.588Z2;1H^[[2;1H^[[2;1H
7882026-01-17T15:12:40.629Z^[[3;1H^[[3;1HBIOS Version: MZ32-AR0-
7892026-01-17T15:12:40.669Z0-M06 ^[[4;1H^[[4
7902026-01-17T15:12:40.753Z1HBIOS Date: 07/10/2021 ^[
7912026-01-17T15:12:40.797Z5;1H^[[5;1HProcessor Type: AMD EPYC 7413 24-Core Processor
7922026-01-17T15:12:40.838Z^[[6;1H^[[6;1HProcessor Speed: 2650 MHz
7932026-01-17T15:12:40.887Z^[[7;1H^[[7;1HTot
7942026-01-17T15:12:40.932Zl Memory: 131072MB (DDR4 3200) ^[
7952026-01-17T15:12:40.976Z8;1H^[[8;1HBMC Firmware Version: 12.54.05
7962026-01-17T15:12:41.022Z^[[9;1H^[[9;1HBMC IP : 172.20.4.56
7972026-01-17T15:12:41.067Z^[[10;1H
7982026-01-17T15:12:41.112Z^[
7992026-01-17T15:12:41.161Z11;1H^[[11;1H
8002026-01-17T15:12:41.201Z^[[12;1H^[[12;1HUSB Devices total: 0 KBDs
8012026-01-17T15:12:41.246Z0 MICE, 0 MASS, 3 HUBs ^[[13;1H^[[13
8022026-01-17T15:12:41.290Z1H ^[
8032026-01-17T15:12:41.495Z14;1H^[[14;1HDetected ATA/ATAPI Devices...
8042026-01-17T15:12:41.764Z^[[15;1H^[[15;1H
8052026-01-17T15:12:41.820Z^[[16;1H^[[16
8062026-01-17T15:12:41.876Z1HDetected NVME Devices... ^[
8072026-01-17T15:12:41.910Z17;1H^[[17;1HHP SSD EX950 1TB Size: 1024.2GB
8082026-01-17T15:12:41.944Z^[[18;1H^[[18;1HHP SSD EX950 1TB Size: 10
8092026-01-17T15:12:44.893Z4.2GB ^[[19;1H^[[19
8102026-01-17T15:12:45.216Z1HPreparing Bios Setup configuration.... ^[
8112026-01-17T15:12:45.250Z20;1H^[[20;1HImporting Bios settings from BMC.
8122026-01-17T15:12:45.865Z^[[21;1H^[[21;1H
8132026-01-17T15:12:59.328Z^[[2J^[[1;1H^[
8142026-01-17T15:12:59.648Z2J^[[1;1H^[[2J^[[1;1H^[[01;00H
8152026-01-17T15:13:17.346ZiPXE initialising devices...ok
8162026-01-17T15:13:17.380Z^[[0m
8172026-01-17T15:13:17.414Z^[[1miPXE 1.21.1+ (g8bff5)^[[0m -- Open Source Network Boot Firmwar
8182026-01-17T15:13:17.446Z-- ^[[36mhttp://ipxe.org^[[0m
8192026-01-17T15:13:17.480ZFeatures: DNS HTTP iSCSI TFTP SRP AoE ELF MBOOT PXE bzImage Menu PXEXT
8202026-01-17T15:13:18.679Znet0: 18:c0:4d:0d:a0:2a using i350 on 0000:c3:00.0 (Ethernet) [open]
8212026-01-17T15:13:18.720Z[Link:up, TX:0 TXE:1 RX:0 RXE:0]
8222026-01-17T15:13:18.765Z[TXE: 1 x "Network unreachable (http://ipxe.org/28086011)"]
8232026-01-17T15:13:22.555ZConfiguring (net0 18:c0:4d:0d:a0:2a)...... ok
8242026-01-17T15:13:22.588Znet0: 10.151.2.100/255.255.255.0 gw 10.151.2.1
8252026-01-17T15:13:22.622ZFilename: http://172.20.0.5:9969/ipxe/sock
8262026-01-17T15:13:22.655Zhttp://172.20.0.5:9969/ipxe/sock... ok
8272026-01-17T15:13:22.688Zsock : 555 bytes [script]
8282026-01-17T15:15:22.664Zworker failed without completing job
8292026-01-17T15:15:22.667Ztask 2 was incomplete, marked failed