01KN2TTPBM6X80H3P84ZP3WT8D: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.39",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KN2TVVCT1R10EF7CJNBTE8YF

Tags:

Output:

SEQ GLOBAL TIME DETAILS
12026-03-31T21:13:27.496Zjob dependencies complete; ready to run (waiting for 18 m 7 s)
22026-03-31T22:06:02.814Zjob assigned to worker 01KN2YVJ56M0SVFNH08RQ9TV8Q [factory lab, sock/16523] (queued for 52 m 35 s)
32026-03-31T22:06:03.018Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-03-31T22:06:03.060Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52026-03-31T22:06:03.116Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62026-03-31T22:06:03.170Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72026-03-31T22:06:03.239Zhttp://172.20.0.5:9969/postboot/sock... ok
82026-03-31T22:06:03.295ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92026-03-31T22:06:03.351ZWARNING: cpu0: failed to update microcode from version 0xa001133 to 0xa0011de
102026-03-31T22:06:03.406ZWARNING: cpu0: using older fallback microcode; update the system firmware
112026-03-31T22:06:03.446Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d5
122026-03-31T22:06:03.503ZOxide Helios Version helios-2.0.23743 64-bit
132026-03-31T22:06:03.547ZConfiguring devices.
142026-03-31T22:06:03.607Z##################################################
152026-03-31T22:06:03.647Z#### ############# ##
162026-03-31T22:06:03.684Z### ### ############ ##
172026-03-31T22:06:03.723Z## ### # ## ### ## Oxide Computer Company ##
182026-03-31T22:06:03.807Z## ## ## ### # ### ##
192026-03-31T22:06:03.865Z## # ### #### #### This Station Under ##
202026-03-31T22:06:03.909Z### ### #### # ### Computer Control ##
212026-03-31T22:06:03.970Z#### #### ### ## ##
222026-03-31T22:06:04.007Z##################################################
232026-03-31T22:06:04.040ZHostname: sock
242026-03-31T22:06:04.092ZLoading smf(7) service descriptions:
252026-03-31T22:06:05.068Zdownloading input: /input/package/work/package.tar.gz
262026-03-31T22:12:56.066Zdownloaded input: /input/package/work/package.tar.gz
 
272026-03-31T22:12:56.091Zstarting task 0: "setup"
282026-03-31T22:12:56.091Z++ uname -s
292026-03-31T22:12:56.091Z+ kern=SunOS
302026-03-31T22:12:56.091Z+ build_user=build
312026-03-31T22:12:56.091Z+ build_uid=12345
322026-03-31T22:12:56.091Z+ work_dir=/work
332026-03-31T22:12:56.091Z+ input_dir=/input
342026-03-31T22:12:56.092Z+ [[ 0 == 12345 ]]
352026-03-31T22:12:56.092Z+ case "$kern" in
362026-03-31T22:12:56.092Z+ groupadd -g 12345 build
372026-03-31T22:12:56.092Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-03-31T22:12:58.085Z+ zfs create -o mountpoint=/work rpool/work
392026-03-31T22:12:58.110Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-03-31T22:12:58.110Z+ home_fs=zfs
412026-03-31T22:12:58.110Z+ [[ zfs == autofs ]]
422026-03-31T22:12:58.110Z+ mkdir -p /home/build
432026-03-31T22:12:58.110Z+ chown build:build /home/build /work
442026-03-31T22:13:00.122Z+ chmod 0700 /home/build /work
452026-03-31T22:13:00.148Zprocess exited: duration 4049 ms, exit code 0
 
462026-03-31T22:13:00.200Zstarting task 1: "authentication"
472026-03-31T22:13:00.246Zprocess exited: duration 46 ms, exit code 0
 
482026-03-31T22:13:00.295Zstarting task 2: "build"
492026-03-31T22:13:00.320Z+ trap _exit_trap EXIT
502026-03-31T22:13:00.320Z+ OPTE_COMMIT=
512026-03-31T22:13:00.320Z+ [[ x != \x ]]
522026-03-31T22:13:00.321Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
532026-03-31T22:13:00.411Zsecpolicy_fs_linkdir: 0x55 = 0x55
542026-03-31T22:13:00.411Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
552026-03-31T22:13:00.411Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
562026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+3: 0 = 0xe5
572026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+4: 0 = 0xb8
582026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+5: 0 = 0x1
592026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
602026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
612026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
622026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
632026-03-31T22:13:00.412Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
642026-03-31T22:13:00.436Z+ [[ -d /opt/oxide ]]
652026-03-31T22:13:00.436Z+ pfexec mv /opt/oxide /opt/oxide-underneath
662026-03-31T22:13:00.436Z+ pfexec mkdir /opt/oxide
672026-03-31T22:13:00.436Z+ pfexec mount -F tmpfs -O swap /opt/oxide
682026-03-31T22:13:00.436Z+ [[ -d /opt/oxide-underneath ]]
692026-03-31T22:13:00.436Z+ cd /opt/oxide-underneath
702026-03-31T22:13:00.437Z+ pfexec tar ceEp@/f - .
712026-03-31T22:13:00.437Z+ cd /opt/oxide
722026-03-31T22:13:00.437Z+ pfexec tar xveEp@/f -
732026-03-31T22:13:00.476Zx ., 0 bytes, 0 tape blocks
742026-03-31T22:13:00.476Zx ./opte, 0 bytes, 0 tape blocks
752026-03-31T22:13:00.477Zx ./opte/bin, 0 bytes, 0 tape blocks
762026-03-31T22:13:00.477Zx ./opte/bin/opteadm, 22792392 bytes, 44517 tape blocks
772026-03-31T22:13:00.501Z+ pfexec rm -rf /opt/oxide-underneath
782026-03-31T22:13:00.501Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
792026-03-31T22:13:00.525Z+ pfexec mkdir /opt/oxide/work
802026-03-31T22:13:00.525Z+ pfexec chown build:build /opt/oxide/work
812026-03-31T22:13:00.525Z+ cd /opt/oxide/work
822026-03-31T22:13:00.525Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
832026-03-31T22:13:21.463ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
842026-03-31T22:13:21.463Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
852026-03-31T22:13:21.463Zx out/target/test, 107 bytes, 1 tape blocks
862026-03-31T22:13:21.463Zx out/npuzone/npuzone, 28059200 bytes, 54804 tape blocks
872026-03-31T22:13:21.463Zx package-manifest.toml, 34895 bytes, 69 tape blocks
882026-03-31T22:13:21.463Zx smf/sled-agent/non-gimlet/config.toml, 5328 bytes, 11 tape blocks
892026-03-31T22:13:21.463Zx target/release/omicron-package, 46989608 bytes, 91777 tape blocks
902026-03-31T22:13:21.463Zx target/release/xtask, 11713264 bytes, 22878 tape blocks
912026-03-31T22:13:21.463Zx target/debug/bootstrap, 347023280 bytes, 677780 tape blocks
922026-03-31T22:13:21.463Zx tests/bootstrap-ae677418c16c6843, 32061232 bytes, 62620 tape blocks
932026-03-31T22:13:21.463Zx tests/commtest-bc7afc413043124b, 33172192 bytes, 64790 tape blocks
942026-03-31T22:13:21.463Zx tests/dhcp_server-2a90c53dd433ac59, 32061248 bytes, 62620 tape blocks
952026-03-31T22:13:21.463Zx tests/end_to_end_tests-e50494250465b102, 390257168 bytes, 762222 tape blocks
962026-03-31T22:13:21.463Zx out/clickhouse.tar.gz, 316983322 bytes, 619109 tape blocks
972026-03-31T22:13:21.463Zx out/clickhouse_keeper.tar.gz, 302061243 bytes, 589964 tape blocks
982026-03-31T22:13:21.463Zx out/clickhouse_server.tar.gz, 316981642 bytes, 619105 tape blocks
992026-03-31T22:13:21.463Zx out/cockroachdb.tar.gz, 161481804 bytes, 315395 tape blocks
1002026-03-31T22:13:21.464Zx out/crucible-pantry-zone.tar.gz, 41458548 bytes, 80974 tape blocks
1012026-03-31T22:13:21.464Zx out/crucible-utils.tar, 18169344 bytes, 35487 tape blocks
1022026-03-31T22:13:21.464Zx out/crucible-zone.tar.gz, 55935971 bytes, 109250 tape blocks
1032026-03-31T22:13:21.464Zx out/external-dns.tar.gz, 46248607 bytes, 90330 tape blocks
1042026-03-31T22:13:21.464Zx out/internal-dns.tar.gz, 46248203 bytes, 90329 tape blocks
1052026-03-31T22:13:21.464Zx out/mg-ddm-gz.tar, 80258048 bytes, 156754 tape blocks
1062026-03-31T22:13:21.464Zx out/nexus.tar.gz, 220815971 bytes, 431282 tape blocks
1072026-03-31T22:13:21.464Zx out/ntp.tar.gz, 27599435 bytes, 53906 tape blocks
1082026-03-31T22:13:21.464Zx out/omicron-gateway-softnpu.tar.gz, 29149382 bytes, 56933 tape blocks
1092026-03-31T22:13:21.464Zx out/omicron-sled-agent.tar, 200027136 bytes, 390678 tape blocks
1102026-03-31T22:13:21.464Zx out/overlay.tar.gz, 3030 bytes, 6 tape blocks
1112026-03-31T22:13:21.464Zx out/oximeter.tar.gz, 61231681 bytes, 119594 tape blocks
1122026-03-31T22:13:21.464Zx out/oxlog.tar, 3872256 bytes, 7563 tape blocks
1132026-03-31T22:13:21.464Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1142026-03-31T22:13:21.464Zx out/propolis-server.tar.gz, 34046668 bytes, 66498 tape blocks
1152026-03-31T22:13:21.464Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1162026-03-31T22:13:21.464Zx out/switch-softnpu.tar.gz, 380094846 bytes, 742373 tape blocks
1172026-03-31T22:13:21.488Z
1182026-03-31T22:13:21.488Zreal 20.933086077
1192026-03-31T22:13:21.488Zuser 18.751571422
1202026-03-31T22:13:21.488Zsys 3.614250438
1212026-03-31T22:13:21.488Ztrap 0.000847992
1222026-03-31T22:13:21.488Ztflt 0.000000000
1232026-03-31T22:13:21.488Zdflt 0.000000000
1242026-03-31T22:13:21.488Zkflt 0.000000000
1252026-03-31T22:13:21.488Zlock 0.000000000
1262026-03-31T22:13:21.488Zslp 18.930423643
1272026-03-31T22:13:21.488Zlat 0.566668160
1282026-03-31T22:13:21.488Zstop 0.000140142
1292026-03-31T22:13:21.488Z+ source .github/buildomat/ci-env.sh
1302026-03-31T22:13:21.488Z++ export CARGO_TERM_COLOR=always
1312026-03-31T22:13:21.488Z++ CARGO_TERM_COLOR=always
1322026-03-31T22:13:21.488Z+ bmat address ls
1332026-03-31T22:13:21.488ZNAME CIDR FIRST LAST COUNT
1342026-03-31T22:13:21.488Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1352026-03-31T22:13:21.488Z++ bmat address ls -f extra -Ho first,last,count
1362026-03-31T22:13:21.488Z+ set -- 10.151.2.164 10.151.2.227 64
1372026-03-31T22:13:21.488Z+ EXTRA_IP_START=10.151.2.164
1382026-03-31T22:13:21.488Z+ EXTRA_IP_END=10.151.2.227
1392026-03-31T22:13:21.488Z+ EXTRA_IP_COUNT=64
1402026-03-31T22:13:21.488Z+ (( EXTRA_IP_COUNT >= 32 ))
1412026-03-31T22:13:21.488Z+ EXTRA_IP_BASE=10.151.2
1422026-03-31T22:13:21.488Z+ EXTRA_IP_FOCTET=164
1432026-03-31T22:13:21.488Z+ EXTRA_IP_LOCTET=227
1442026-03-31T22:13:21.488Z+ SERVICE_IP_POOL_START=10.151.2.164
1452026-03-31T22:13:21.488Z+ SERVICE_IP_POOL_END=10.151.2.173
1462026-03-31T22:13:21.489Z+ DNS_IP1=10.151.2.164
1472026-03-31T22:13:21.489Z+ DNS_IP2=10.151.2.165
1482026-03-31T22:13:21.489Z+ UPLINK_IP=10.151.2.174
1492026-03-31T22:13:21.489Z+ IPPOOL_START=10.151.2.175
1502026-03-31T22:13:21.489Z+ IPPOOL_END=10.151.2.227
1512026-03-31T22:13:21.489Z++ ipadm show-addr -po type,addr
1522026-03-31T22:13:21.489Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1532026-03-31T22:13:21.489Z+ GATEWAY_IP=10.151.2.100
1542026-03-31T22:13:21.489Z+ [[ -n 10.151.2.100 ]]
1552026-03-31T22:13:21.489Z+ ping -s 10.151.2.100 56 1
1562026-03-31T22:13:21.489ZPING 10.151.2.100: 56 data bytes
1572026-03-31T22:13:21.489Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.011 ms
1582026-03-31T22:13:21.489Z
1592026-03-31T22:13:21.489Z----10.151.2.100 PING Statistics----
1602026-03-31T22:13:21.489Z1 packets transmitted, 1 packets received, 0% packet loss
1612026-03-31T22:13:21.489Zround-trip (ms) min/avg/max/stddev = 0.011/0.011/0.011/-nan
1622026-03-31T22:13:21.489Z++ arp -an
1632026-03-31T22:13:21.489Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1642026-03-31T22:13:21.514Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1652026-03-31T22:13:21.514Z+ routeadm -e ipv4-forwarding -u
1662026-03-31T22:13:21.585Z+ PXA_START=10.151.2.164
1672026-03-31T22:13:21.586Z+ PXA_END=10.151.2.227
1682026-03-31T22:13:21.586Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
1692026-03-31T22:13:21.799Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1702026-03-31T22:13:21.822ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1712026-03-31T22:13:21.872Zcreating virtual hardware
1722026-03-31T22:13:21.896Zcreating /scratch/m2_0.vdev
1732026-03-31T22:13:21.896Zcreating /scratch/m2_1.vdev
1742026-03-31T22:13:21.896Zcreating /scratch/u2_0.vdev
1752026-03-31T22:13:21.896Zcreating /scratch/u2_1.vdev
1762026-03-31T22:13:21.896Zcreating /scratch/u2_2.vdev
1772026-03-31T22:13:21.896Zcreating /scratch/u2_3.vdev
1782026-03-31T22:13:21.896Zcreating /scratch/u2_4.vdev
1792026-03-31T22:13:21.896Zcreating /scratch/u2_5.vdev
1802026-03-31T22:13:21.896Zcreating /scratch/u2_6.vdev
1812026-03-31T22:13:21.896Zcreating /scratch/u2_7.vdev
1822026-03-31T22:13:21.896Zcreating /scratch/u2_8.vdev
1832026-03-31T22:13:21.920ZSimnet net0/sc0_0 exists
1842026-03-31T22:13:21.944ZSimnet net1/sc1_0 exists
1852026-03-31T22:13:21.944ZVnic sc0_1 exists
1862026-03-31T22:13:34.624ZUsing 10.151.2.100 as gateway ip
1872026-03-31T22:13:34.624Zusing 18:C0:4D:0D:A0:2A as gateway mac
1882026-03-31T22:13:34.624Zconfiguring SoftNPU ARP entry
1892026-03-31T22:13:34.649Zconfiguring SoftNPU proxy ARP
1902026-03-31T22:13:34.699ZSoftNPU state:
1912026-03-31T22:13:34.699Z local v6:
1922026-03-31T22:13:34.699Z local v4:
1932026-03-31T22:13:34.699Z router v6_idx:
1942026-03-31T22:13:34.699Z router v6_routes:
1952026-03-31T22:13:34.699Z router v4_idx:
1962026-03-31T22:13:34.700Z router v4_routes:
1972026-03-31T22:13:34.700Z resolver v4:
1982026-03-31T22:13:34.700Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
1992026-03-31T22:13:34.700Z resolver v6:
2002026-03-31T22:13:34.700Z nat v4:
2012026-03-31T22:13:34.700Z nat_v6:
2022026-03-31T22:13:34.700Z attached subnet v4:
2032026-03-31T22:13:34.700Z attached subnet v6:
2042026-03-31T22:13:34.700Z port_mac:
2052026-03-31T22:13:34.700Z proxy arp:
2062026-03-31T22:13:34.700Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2072026-03-31T22:13:34.700Zcreated virtual hardware
2082026-03-31T22:13:34.724Z
2092026-03-31T22:13:34.724Zreal 12.898012621
2102026-03-31T22:13:34.724Zuser 0.665118732
2112026-03-31T22:13:34.724Zsys 3.333418406
2122026-03-31T22:13:34.724Ztrap 0.000077726
2132026-03-31T22:13:34.724Ztflt 0.000179656
2142026-03-31T22:13:34.724Zdflt 0.000407040
2152026-03-31T22:13:34.724Zkflt 0.000003709
2162026-03-31T22:13:34.724Zlock 9:59.392332280
2172026-03-31T22:13:34.724Zslp 38.806122831
2182026-03-31T22:13:34.724Zlat 0.127062398
2192026-03-31T22:13:34.724Zstop 0.245819737
2202026-03-31T22:13:34.724Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2212026-03-31T22:13:34.755Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2222026-03-31T22:13:34.800Z+ diff -u pkg/config.toml~ pkg/config.toml
2232026-03-31T22:13:34.800Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2242026-03-31T22:13:34.800Z+++ pkg/config.toml Tue Mar 31 22:13:34 2026
2252026-03-31T22:13:34.800Z@@ -29,18 +29,18 @@
2262026-03-31T22:13:34.800Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2272026-03-31T22:13:34.800Z # followed by a string that is embedded into their fake serial values.
2282026-03-31T22:13:34.800Z vdevs = [
2292026-03-31T22:13:34.800Z- "m2_0.vdev",
2302026-03-31T22:13:34.800Z- "m2_1.vdev",
2312026-03-31T22:13:34.800Z+ "/scratch/m2_0.vdev",
2322026-03-31T22:13:34.800Z+ "/scratch/m2_1.vdev",
2332026-03-31T22:13:34.800Z
2342026-03-31T22:13:34.800Z- "u2_0.vdev",
2352026-03-31T22:13:34.800Z- "u2_1.vdev",
2362026-03-31T22:13:34.800Z- "u2_2.vdev",
2372026-03-31T22:13:34.800Z- "u2_3.vdev",
2382026-03-31T22:13:34.800Z- "u2_4.vdev",
2392026-03-31T22:13:34.800Z- "u2_5.vdev",
2402026-03-31T22:13:34.800Z- "u2_6.vdev",
2412026-03-31T22:13:34.800Z- "u2_7.vdev",
2422026-03-31T22:13:34.800Z- "u2_8.vdev",
2432026-03-31T22:13:34.800Z+ "/scratch/u2_0.vdev",
2442026-03-31T22:13:34.800Z+ "/scratch/u2_1.vdev",
2452026-03-31T22:13:34.800Z+ "/scratch/u2_2.vdev",
2462026-03-31T22:13:34.800Z+ "/scratch/u2_3.vdev",
2472026-03-31T22:13:34.800Z+ "/scratch/u2_4.vdev",
2482026-03-31T22:13:34.800Z+ "/scratch/u2_5.vdev",
2492026-03-31T22:13:34.800Z+ "/scratch/u2_6.vdev",
2502026-03-31T22:13:34.800Z+ "/scratch/u2_7.vdev",
2512026-03-31T22:13:34.800Z+ "/scratch/u2_8.vdev",
2522026-03-31T22:13:34.801Z ]
2532026-03-31T22:13:34.801Z
2542026-03-31T22:13:34.801Z # The amount of memory held back for services which exist between zero and one
2552026-03-31T22:13:34.801Z+ true
2562026-03-31T22:13:34.801Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2572026-03-31T22:13:34.801Z+ EXPECTED_ZPOOL_COUNT=9
2582026-03-31T22:13:34.801Z+ echo 'expected number of zpools is 9'
2592026-03-31T22:13:34.801Zexpected number of zpools is 9
2602026-03-31T22:13:34.801Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2612026-03-31T22:13:34.801Z+ SILO_NAME=recovery
2622026-03-31T22:13:34.801Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2632026-03-31T22:13:34.801Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2642026-03-31T22:13:34.801Z+ sed -i~ '
2652026-03-31T22:13:34.801Z /^external_dns_ips/c\
2662026-03-31T22:13:34.801Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2672026-03-31T22:13:34.801Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2682026-03-31T22:13:34.801Z /^first/c\
2692026-03-31T22:13:34.801Zfirst = "10.151.2.164"
2702026-03-31T22:13:34.801Z /^last/c\
2712026-03-31T22:13:34.801Zlast = "10.151.2.173"
2722026-03-31T22:13:34.801Z }
2732026-03-31T22:13:34.801Z /^infra_ip_first/c\
2742026-03-31T22:13:34.801Zinfra_ip_first = "10.151.2.174"
2752026-03-31T22:13:34.801Z /^infra_ip_last/c\
2762026-03-31T22:13:34.801Zinfra_ip_last = "10.151.2.174"
2772026-03-31T22:13:34.801Z /^\[\[rack_network_config.ports/,/^$/ {
2782026-03-31T22:13:34.801Z /^routes/c\
2792026-03-31T22:13:34.801Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2802026-03-31T22:13:34.801Z /^addresses/c\
2812026-03-31T22:13:34.801Zaddresses = \[{address = {type = "static", ip_net = "10.151.2.174/24"} }\]
2822026-03-31T22:13:34.801Z }
2832026-03-31T22:13:34.801Z' pkg/config-rss.toml
2842026-03-31T22:13:34.801Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2852026-03-31T22:13:34.801Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2862026-03-31T22:13:34.801Z+++ pkg/config-rss.toml Tue Mar 31 22:13:34 2026
2872026-03-31T22:13:34.801Z@@ -33,7 +33,7 @@
2882026-03-31T22:13:34.801Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2892026-03-31T22:13:34.801Z # must be contained in one of the "internal services" IP Pool ranges listed
2902026-03-31T22:13:34.802Z # below.
2912026-03-31T22:13:34.802Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2922026-03-31T22:13:34.802Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2932026-03-31T22:13:34.802Z
2942026-03-31T22:13:34.802Z # Initial TLS certificates for the external API
2952026-03-31T22:13:34.802Z #
2962026-03-31T22:13:34.802Z@@ -69,8 +69,8 @@
2972026-03-31T22:13:34.802Z #
2982026-03-31T22:13:34.802Z # For more on this and what to put here, see docs/how-to-run.adoc.
2992026-03-31T22:13:34.802Z [[internal_services_ip_pool_ranges]]
3002026-03-31T22:13:34.802Z-first = "192.168.1.20"
3012026-03-31T22:13:34.802Z-last = "192.168.1.29"
3022026-03-31T22:13:34.802Z+first = "10.151.2.164"
3032026-03-31T22:13:34.802Z+last = "10.151.2.173"
3042026-03-31T22:13:34.802Z
3052026-03-31T22:13:34.802Z # TODO - this configuration is subject to change going forward. Ultimately these
3062026-03-31T22:13:34.802Z # parameters should be provided to the control plane via wicket, but we need to
3072026-03-31T22:13:34.802Z@@ -91,8 +91,8 @@
3082026-03-31T22:13:34.802Z # A range of IP addresses used by Boundary Services on the external network. In
3092026-03-31T22:13:34.802Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3102026-03-31T22:13:34.802Z # With softnpu, only one address is used.
3112026-03-31T22:13:34.802Z-infra_ip_first = "192.168.1.30"
3122026-03-31T22:13:34.802Z-infra_ip_last = "192.168.1.30"
3132026-03-31T22:13:34.802Z+infra_ip_first = "10.151.2.174"
3142026-03-31T22:13:34.802Z+infra_ip_last = "10.151.2.174"
3152026-03-31T22:13:34.802Z
3162026-03-31T22:13:34.802Z # Configurations for BGP routers to run on the scrimlets.
3172026-03-31T22:13:34.802Z bgp = []
3182026-03-31T22:13:34.802Z@@ -100,9 +100,9 @@
3192026-03-31T22:13:34.802Z # You can configure multiple uplinks by repeating the following stanza
3202026-03-31T22:13:34.802Z [[rack_network_config.ports]]
3212026-03-31T22:13:34.802Z # Routes associated with this port.
3222026-03-31T22:13:34.826Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3232026-03-31T22:13:34.826Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3242026-03-31T22:13:34.826Z # Addresses associated with this port.
3252026-03-31T22:13:34.826Z-addresses = [{address = {type = "static", ip_net = "192.168.1.30/24"}}]
3262026-03-31T22:13:34.827Z+addresses = [{address = {type = "static", ip_net = "10.151.2.174/24"} }]
3272026-03-31T22:13:34.827Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3282026-03-31T22:13:34.827Z port = "qsfp0"
3292026-03-31T22:13:34.827Z # The speed of this port.
3302026-03-31T22:13:34.827Z+ true
3312026-03-31T22:13:34.827Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3322026-03-31T22:13:34.827Za pkg/config-rss.toml 8K
3332026-03-31T22:13:34.827Za pkg/config.toml 6K
3342026-03-31T22:13:34.827Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3352026-03-31T22:13:34.827Z+ TLS_NAME=recovery.sys.oxide.test
3362026-03-31T22:13:34.827Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3372026-03-31T22:13:34.968Z...+.+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*......+.+..+...+..........+.....+..........+...+.........+.....+...+.......+...+..+......+.+...+...+..+.......+......+..+.+.....+....+........+....+.....+.+...+.........+..+.+.....+.............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.......+......+.......+..+...+...+............+.......+...+...+..+.+............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3382026-03-31T22:13:36.248Z..+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*........+.....+...+.......+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.....+..........+..............+...+.+.................+.............+..+...+...................+............+...+.....+.+....................+......+...+......+....+...............+...+...............+...+.........+...........................+...+..+...............+.........+.+...+.....+.......+.....+....+.........+......+......+...........+.......+...+..+.............+..+............+...+......................+..................+.................+......+............+.......+......+..+......+...............+....+..+......+........................+...+..................+.......+...+.....+....+...+......+......+...........+....+...+...............+.................+...+..........+..+.............+...+..+......+....+......+...........+.+...........................+..+......+............+.+..+.......+..+....+......+...+...+........+.+........+............+....+..+....+......+.........+.....+.+..............+.+...+..............+...+....+..................+...........+......+...............+.+..+...+.......+........+...................+...+.....+......+.......+........+......+...................+......+..............+.+.....+.............+...+..+.......+.....+...+..........+......+..+....+.....+...+......+..........+.....+.+.....+......+.........+.+............+..+.+..+...............+......+.........+...+..........+...............+.........+....................+....+.....+............+.............+.........+........+..........+...............+...+...........+...............+.+...+............+............+.....+.+.........+........+...+...............+.+........+...+..........+..+......+...................+............+.....+......+.......+............+......+.................+.....................+......+......+.......+......+......+..........................+....+........+............+..................+...+...+....+......+.....+......+...............+................+...+.........+.....+.+......+.........+.....+......................+.....+............+.+..+.............+.........+........+..........+...+......+.....+...+.+......+......+.....+.............+.........+......+..+....+........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3392026-03-31T22:13:36.273Z-----
3402026-03-31T22:13:36.297Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3412026-03-31T22:13:36.297Za pkg/initial-tls-cert.pem 2K
3422026-03-31T22:13:36.297Za pkg/initial-tls-key.pem 4K
3432026-03-31T22:13:36.297Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3442026-03-31T22:13:36.297Z+ rmdir pkg
3452026-03-31T22:13:36.297Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3462026-03-31T22:13:36.297Z+ pfexec mkdir -p /usr/oxide
3472026-03-31T22:13:36.297Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3482026-03-31T22:13:36.914Z+ pfexec chmod +x /usr/oxide/oxide
3492026-03-31T22:13:36.939Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3502026-03-31T22:13:55.219Z+ OMICRON_NO_UNINSTALL=1
3512026-03-31T22:13:55.219Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3522026-03-31T22:13:55.242ZLogging to: /opt/oxide/work/out/LOG
3532026-03-31T22:13:56.068Z
3542026-03-31T22:13:56.068Zreal 0.840052192
3552026-03-31T22:13:56.068Zuser 0.167391687
3562026-03-31T22:13:56.094Zsys 4.822228757
3572026-03-31T22:13:56.094Ztrap 0.000002060
3582026-03-31T22:13:56.094Ztflt 0.000020910
3592026-03-31T22:13:56.094Zdflt 0.000399986
3602026-03-31T22:13:56.094Zkflt 0.000000000
3612026-03-31T22:13:56.094Zlock 1:15.855263120
3622026-03-31T22:13:56.094Zslp 1.038721813
3632026-03-31T22:13:56.095Zlat 0.037371801
3642026-03-31T22:13:56.095Zstop 0.062057627
3652026-03-31T22:13:56.095Z+ retry=0
3662026-03-31T22:13:56.095Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3672026-03-31T22:13:56.095Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3682026-03-31T22:13:56.095Z+ [[ 0 -gt 30 ]]
3692026-03-31T22:13:56.095Z+ sleep 1
3702026-03-31T22:13:57.079Z+ retry=1
3712026-03-31T22:13:57.079Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3722026-03-31T22:13:57.102Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3732026-03-31T22:13:57.102Z+ [[ 1 -gt 30 ]]
3742026-03-31T22:13:57.102Z+ sleep 1
3752026-03-31T22:13:58.092Z+ retry=2
3762026-03-31T22:13:58.092Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3772026-03-31T22:13:58.119Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3782026-03-31T22:13:58.119Z+ [[ 2 -gt 30 ]]
3792026-03-31T22:13:58.119Z+ sleep 1
3802026-03-31T22:13:59.107Z+ retry=3
3812026-03-31T22:13:59.107Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3822026-03-31T22:14:00.118Zcurl: (28) Connection timed out after 1000 milliseconds
3832026-03-31T22:14:00.141Z+ [[ 3 -gt 30 ]]
3842026-03-31T22:14:00.141Z+ sleep 1
3852026-03-31T22:14:01.123Z+ retry=4
3862026-03-31T22:14:01.123Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3872026-03-31T22:14:02.136Zcurl: (28) Connection timed out after 1000 milliseconds
3882026-03-31T22:14:02.159Z+ [[ 4 -gt 30 ]]
3892026-03-31T22:14:02.159Z+ sleep 1
3902026-03-31T22:14:03.141Z+ retry=5
3912026-03-31T22:14:03.141Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3922026-03-31T22:14:04.150Zcurl: (28) Connection timed out after 1000 milliseconds
3932026-03-31T22:14:04.174Z+ [[ 5 -gt 30 ]]
3942026-03-31T22:14:04.174Z+ sleep 1
3952026-03-31T22:14:05.155Z+ retry=6
3962026-03-31T22:14:05.155Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3972026-03-31T22:14:06.168Zcurl: (28) Connection timed out after 1000 milliseconds
3982026-03-31T22:14:06.192Z+ [[ 6 -gt 30 ]]
3992026-03-31T22:14:06.192Z+ sleep 1
4002026-03-31T22:14:07.173Z+ retry=7
4012026-03-31T22:14:07.173Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4022026-03-31T22:14:08.185Zcurl: (28) Connection timed out after 1000 milliseconds
4032026-03-31T22:14:08.208Z+ [[ 7 -gt 30 ]]
4042026-03-31T22:14:08.208Z+ sleep 1
4052026-03-31T22:14:09.191Z+ retry=8
4062026-03-31T22:14:09.191Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4072026-03-31T22:14:10.208Zcurl: (28) Connection timed out after 1000 milliseconds
4082026-03-31T22:14:10.232Z+ [[ 8 -gt 30 ]]
4092026-03-31T22:14:10.232Z+ sleep 1
4102026-03-31T22:14:11.213Z+ retry=9
4112026-03-31T22:14:11.213Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4122026-03-31T22:14:12.223Zcurl: (28) Connection timed out after 1000 milliseconds
4132026-03-31T22:14:12.247Z+ [[ 9 -gt 30 ]]
4142026-03-31T22:14:12.247Z+ sleep 1
4152026-03-31T22:14:13.229Z+ retry=10
4162026-03-31T22:14:13.229Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4172026-03-31T22:14:14.458Zcurl: (28) Connection timed out after 1000 milliseconds
4182026-03-31T22:14:14.482Z+ [[ 10 -gt 30 ]]
4192026-03-31T22:14:14.482Z+ sleep 1
4202026-03-31T22:14:15.487Z+ retry=11
4212026-03-31T22:14:15.487Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4222026-03-31T22:14:16.506Zcurl: (28) Connection timed out after 1000 milliseconds
4232026-03-31T22:14:16.530Z+ [[ 11 -gt 30 ]]
4242026-03-31T22:14:16.530Z+ sleep 1
4252026-03-31T22:14:17.512Z+ retry=12
4262026-03-31T22:14:17.512Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4272026-03-31T22:14:18.523Zcurl: (28) Connection timed out after 1000 milliseconds
4282026-03-31T22:14:18.546Z+ [[ 12 -gt 30 ]]
4292026-03-31T22:14:18.546Z+ sleep 1
4302026-03-31T22:14:19.527Z+ retry=13
4312026-03-31T22:14:19.527Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4322026-03-31T22:14:19.551Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
4332026-03-31T22:14:19.575Zlocal v6:
4342026-03-31T22:14:19.575Zfe80::d0ca:1ff:fe14:2fae
4352026-03-31T22:14:19.575Zfd69:2774:9559:5147::1
4362026-03-31T22:14:19.575Zlocal v4:
4372026-03-31T22:14:19.575Zrouter v6_idx:
4382026-03-31T22:14:19.575Zrouter v6_routes:
4392026-03-31T22:14:19.575Zrouter v4_idx:
4402026-03-31T22:14:19.575Zrouter v4_routes:
4412026-03-31T22:14:19.575Zresolver v4:
4422026-03-31T22:14:19.575Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
4432026-03-31T22:14:19.575Zresolver v6:
4442026-03-31T22:14:19.575Zfe80::d0ca:1ff:fe14:2fae -> d2:ca:01:14:2f:ae
4452026-03-31T22:14:19.575Znat v4:
4462026-03-31T22:14:19.575Znat_v6:
4472026-03-31T22:14:19.575Zattached subnet v4:
4482026-03-31T22:14:19.575Zattached subnet v6:
4492026-03-31T22:14:19.575Zport_mac:
4502026-03-31T22:14:19.575Z1000: a8:40:25:49:25:6b
4512026-03-31T22:14:19.575Z1: a8:40:25:49:25:6c
4522026-03-31T22:14:19.575Zproxy arp:
4532026-03-31T22:14:19.575Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
4542026-03-31T22:14:19.575Z+ retry=0
4552026-03-31T22:14:19.603Z+++ zoneadm list -n
4562026-03-31T22:14:19.603Z+++ grep oxz_ntp
4572026-03-31T22:14:19.603Z++ pfexec svcs -z -Hostate oxide/ntp
4582026-03-31T22:14:19.603Zsvcs: zone '-Hostate': entity not found
4592026-03-31T22:14:19.603Z++ true
4602026-03-31T22:14:19.603Z+ [[ '' != online ]]
4612026-03-31T22:14:19.603Z+ [[ 0 -gt 60 ]]
4622026-03-31T22:14:19.603Z+ sleep 1
4632026-03-31T22:14:20.595Z+ retry=1
4642026-03-31T22:14:20.640Z+++ zoneadm list -n
4652026-03-31T22:14:20.640Z+++ grep oxz_ntp
4662026-03-31T22:14:20.640Z++ pfexec svcs -z -Hostate oxide/ntp
4672026-03-31T22:14:20.640Zsvcs: zone '-Hostate': entity not found
4682026-03-31T22:14:20.640Z++ true
4692026-03-31T22:14:20.640Z+ [[ '' != online ]]
4702026-03-31T22:14:20.640Z+ [[ 1 -gt 60 ]]
4712026-03-31T22:14:20.640Z+ sleep 1
4722026-03-31T22:14:21.617Z+ retry=2
4732026-03-31T22:14:21.641Z+++ zoneadm list -n
4742026-03-31T22:14:21.641Z+++ grep oxz_ntp
4752026-03-31T22:14:21.641Z++ pfexec svcs -z -Hostate oxide/ntp
4762026-03-31T22:14:21.641Zsvcs: zone '-Hostate': entity not found
4772026-03-31T22:14:21.641Z++ true
4782026-03-31T22:14:21.641Z+ [[ '' != online ]]
4792026-03-31T22:14:21.641Z+ [[ 2 -gt 60 ]]
4802026-03-31T22:14:21.641Z+ sleep 1
4812026-03-31T22:14:22.638Z+ retry=3
4822026-03-31T22:14:22.662Z+++ zoneadm list -n
4832026-03-31T22:14:22.662Z+++ grep oxz_ntp
4842026-03-31T22:14:22.662Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
4852026-03-31T22:14:22.662Zsvcs: Could not bind to repository server for zone oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9: repository server unavailable
4862026-03-31T22:14:22.662Z++ true
4872026-03-31T22:14:22.662Z+ [[ '' != online ]]
4882026-03-31T22:14:22.662Z+ [[ 3 -gt 60 ]]
4892026-03-31T22:14:22.662Z+ sleep 1
4902026-03-31T22:14:23.661Z+ retry=4
4912026-03-31T22:14:23.684Z+++ zoneadm list -n
4922026-03-31T22:14:23.685Z+++ grep oxz_ntp
4932026-03-31T22:14:23.685Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
4942026-03-31T22:14:23.708Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4952026-03-31T22:14:23.708Z++ true
4962026-03-31T22:14:23.708Z+ [[ '' != online ]]
4972026-03-31T22:14:23.708Z+ [[ 4 -gt 60 ]]
4982026-03-31T22:14:23.708Z+ sleep 1
4992026-03-31T22:14:24.691Z+ retry=5
5002026-03-31T22:14:24.714Z+++ zoneadm list -n
5012026-03-31T22:14:24.714Z+++ grep oxz_ntp
5022026-03-31T22:14:24.714Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5032026-03-31T22:14:24.738Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
5042026-03-31T22:14:24.738Z++ true
5052026-03-31T22:14:24.738Z+ [[ '' != online ]]
5062026-03-31T22:14:24.738Z+ [[ 5 -gt 60 ]]
5072026-03-31T22:14:24.738Z+ sleep 1
5082026-03-31T22:14:25.720Z+ retry=6
5092026-03-31T22:14:25.745Z+++ zoneadm list -n
5102026-03-31T22:14:25.745Z+++ grep oxz_ntp
5112026-03-31T22:14:25.745Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5122026-03-31T22:14:25.745Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
5132026-03-31T22:14:25.745Z++ true
5142026-03-31T22:14:25.769Z+ [[ '' != online ]]
5152026-03-31T22:14:25.769Z+ [[ 6 -gt 60 ]]
5162026-03-31T22:14:25.769Z+ sleep 1
5172026-03-31T22:14:26.748Z+ retry=7
5182026-03-31T22:14:26.771Z+++ zoneadm list -n
5192026-03-31T22:14:26.772Z+++ grep oxz_ntp
5202026-03-31T22:14:26.772Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5212026-03-31T22:14:26.772Z+ [[ offline != online ]]
5222026-03-31T22:14:26.772Z+ [[ 7 -gt 60 ]]
5232026-03-31T22:14:26.772Z+ sleep 1
5242026-03-31T22:14:27.770Z+ retry=8
5252026-03-31T22:14:27.794Z+++ zoneadm list -n
5262026-03-31T22:14:27.794Z+++ grep oxz_ntp
5272026-03-31T22:14:27.794Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5282026-03-31T22:14:27.794Z+ [[ offline != online ]]
5292026-03-31T22:14:27.794Z+ [[ 8 -gt 60 ]]
5302026-03-31T22:14:27.794Z+ sleep 1
5312026-03-31T22:14:28.791Z+ retry=9
5322026-03-31T22:14:28.815Z+++ zoneadm list -n
5332026-03-31T22:14:28.815Z+++ grep oxz_ntp
5342026-03-31T22:14:28.815Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5352026-03-31T22:14:28.815Z+ [[ offline != online ]]
5362026-03-31T22:14:28.815Z+ [[ 9 -gt 60 ]]
5372026-03-31T22:14:28.815Z+ sleep 1
5382026-03-31T22:14:29.813Z+ retry=10
5392026-03-31T22:14:29.838Z+++ zoneadm list -n
5402026-03-31T22:14:29.838Z+++ grep oxz_ntp
5412026-03-31T22:14:29.838Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5422026-03-31T22:14:29.838Z+ [[ offline != online ]]
5432026-03-31T22:14:29.838Z+ [[ 10 -gt 60 ]]
5442026-03-31T22:14:29.838Z+ sleep 1
5452026-03-31T22:14:30.836Z+ retry=11
5462026-03-31T22:14:30.861Z+++ zoneadm list -n
5472026-03-31T22:14:30.861Z+++ grep oxz_ntp
5482026-03-31T22:14:30.861Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5492026-03-31T22:14:30.861Z+ [[ offline != online ]]
5502026-03-31T22:14:30.861Z+ [[ 11 -gt 60 ]]
5512026-03-31T22:14:30.861Z+ sleep 1
5522026-03-31T22:14:31.857Z+ retry=12
5532026-03-31T22:14:31.881Z+++ zoneadm list -n
5542026-03-31T22:14:31.881Z+++ grep oxz_ntp
5552026-03-31T22:14:31.881Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5562026-03-31T22:14:31.881Z+ [[ offline != online ]]
5572026-03-31T22:14:31.881Z+ [[ 12 -gt 60 ]]
5582026-03-31T22:14:31.881Z+ sleep 1
5592026-03-31T22:14:32.879Z+ retry=13
5602026-03-31T22:14:32.903Z+++ zoneadm list -n
5612026-03-31T22:14:32.903Z+++ grep oxz_ntp
5622026-03-31T22:14:32.903Z++ pfexec svcs -z oxz_ntp_982246da-5433-433e-aee0-c7f528bbe2f9 -Hostate oxide/ntp
5632026-03-31T22:14:32.903Z+ [[ online != online ]]
5642026-03-31T22:14:32.903Z+ echo 'Waited for chrony: 13s'
5652026-03-31T22:14:32.903Z+ retry=0
5662026-03-31T22:14:32.903ZWaited for chrony: 13s
5672026-03-31T22:14:32.903Z+ zoneadm list
5682026-03-31T22:14:32.903Z+ grep nexus
5692026-03-31T22:14:32.927Z+ [[ 0 -gt 300 ]]
5702026-03-31T22:14:32.927Z+ sleep 1
5712026-03-31T22:14:33.912Z+ retry=1
5722026-03-31T22:14:33.937Z+ zoneadm list
5732026-03-31T22:14:33.937Z+ grep nexus
5742026-03-31T22:14:33.937Z+ [[ 1 -gt 300 ]]
5752026-03-31T22:14:33.937Z+ sleep 1
5762026-03-31T22:14:34.920Z+ retry=2
5772026-03-31T22:14:34.945Z+ zoneadm list
5782026-03-31T22:14:34.945Z+ grep nexus
5792026-03-31T22:14:34.945Z+ [[ 2 -gt 300 ]]
5802026-03-31T22:14:34.945Z+ sleep 1
5812026-03-31T22:14:35.931Z+ retry=3
5822026-03-31T22:14:35.954Z+ zoneadm list
5832026-03-31T22:14:35.954Z+ grep nexus
5842026-03-31T22:14:35.954Z+ [[ 3 -gt 300 ]]
5852026-03-31T22:14:35.954Z+ sleep 1
5862026-03-31T22:14:36.954Z+ retry=4
5872026-03-31T22:14:36.977Z+ zoneadm list
5882026-03-31T22:14:36.977Z+ grep nexus
5892026-03-31T22:14:36.977Z+ [[ 4 -gt 300 ]]
5902026-03-31T22:14:36.977Z+ sleep 1
5912026-03-31T22:14:37.973Z+ retry=5
5922026-03-31T22:14:37.997Z+ zoneadm list
5932026-03-31T22:14:37.997Z+ grep nexus
5942026-03-31T22:14:37.997Z+ [[ 5 -gt 300 ]]
5952026-03-31T22:14:37.997Z+ sleep 1
5962026-03-31T22:14:39.002Z+ retry=6
5972026-03-31T22:14:39.026Z+ zoneadm list
5982026-03-31T22:14:39.026Z+ grep nexus
5992026-03-31T22:14:39.026Z+ [[ 6 -gt 300 ]]
6002026-03-31T22:14:39.026Z+ sleep 1
6012026-03-31T22:14:40.237Z+ retry=7
6022026-03-31T22:14:40.261Z+ zoneadm list
6032026-03-31T22:14:40.261Z+ grep nexus
6042026-03-31T22:14:40.284Z+ [[ 7 -gt 300 ]]
6052026-03-31T22:14:40.284Z+ sleep 1
6062026-03-31T22:14:41.282Z+ retry=8
6072026-03-31T22:14:41.306Z+ zoneadm list
6082026-03-31T22:14:41.306Z+ grep nexus
6092026-03-31T22:14:41.306Z+ [[ 8 -gt 300 ]]
6102026-03-31T22:14:41.307Z+ sleep 1
6112026-03-31T22:14:42.298Z+ retry=9
6122026-03-31T22:14:42.322Z+ zoneadm list
6132026-03-31T22:14:42.322Z+ grep nexus
6142026-03-31T22:14:42.322Z+ [[ 9 -gt 300 ]]
6152026-03-31T22:14:42.322Z+ sleep 1
6162026-03-31T22:14:43.317Z+ retry=10
6172026-03-31T22:14:43.341Z+ zoneadm list
6182026-03-31T22:14:43.341Z+ grep nexus
6192026-03-31T22:14:43.341Z+ [[ 10 -gt 300 ]]
6202026-03-31T22:14:43.341Z+ sleep 1
6212026-03-31T22:14:44.331Z+ retry=11
6222026-03-31T22:14:44.354Z+ zoneadm list
6232026-03-31T22:14:44.355Z+ grep nexus
6242026-03-31T22:14:44.355Z+ [[ 11 -gt 300 ]]
6252026-03-31T22:14:44.355Z+ sleep 1
6262026-03-31T22:14:45.344Z+ retry=12
6272026-03-31T22:14:45.368Z+ zoneadm list
6282026-03-31T22:14:45.368Z+ grep nexus
6292026-03-31T22:14:45.368Z+ [[ 12 -gt 300 ]]
6302026-03-31T22:14:45.368Z+ sleep 1
6312026-03-31T22:14:46.359Z+ retry=13
6322026-03-31T22:14:46.383Z+ zoneadm list
6332026-03-31T22:14:46.383Z+ grep nexus
6342026-03-31T22:14:46.383Z+ [[ 13 -gt 300 ]]
6352026-03-31T22:14:46.383Z+ sleep 1
6362026-03-31T22:14:47.379Z+ retry=14
6372026-03-31T22:14:47.402Z+ zoneadm list
6382026-03-31T22:14:47.402Z+ grep nexus
6392026-03-31T22:14:47.402Z+ [[ 14 -gt 300 ]]
6402026-03-31T22:14:47.402Z+ sleep 1
6412026-03-31T22:14:48.395Z+ retry=15
6422026-03-31T22:14:48.419Z+ zoneadm list
6432026-03-31T22:14:48.419Z+ grep nexus
6442026-03-31T22:14:48.419Z+ [[ 15 -gt 300 ]]
6452026-03-31T22:14:48.419Z+ sleep 1
 
6462026-03-31T22:14:50.495Zsock console login: WARNING: panicked at lib/opte/src/ddi/mblk.rs:291:14:
6472026-03-31T22:14:50.540Zsomehow failed to get an mblk...
6482026-03-31T22:14:50.611Zpanic[cpu18]/thread=fffffe85e6714060: panicked at lib/opte/src/ddi/mblk.rs:291:14:
6492026-03-31T22:14:50.667Zsomehow failed to get an mblk...
6502026-03-31T22:14:50.701Zfffffe00c6257d40 xde:_RNvCsj0qrnF3AvHz_7___rustc17rust_begin_unwind+16d ()
6512026-03-31T22:14:50.746Zfffffe00c6257d70 xde:opte_panic_debug+3446024b ()
6522026-03-31T22:14:50.790Zfffffe00c6257dd0 xde:opte_panic_debug+34460700 ()
6532026-03-31T22:14:50.835Zfffffe00c6257e80 xde:_ZN4opte6engine6packet8EmitSpec5apply17hea7022e6e50299e3E+e21 ()
6542026-03-31T22:14:50.880Zfffffe00c62583e0 xde:xde_mc_tx+33ce ()
6552026-03-31T22:14:50.947Zfffffe00c6258410 mac:mac_ring_tx+50 ()
6562026-03-31T22:14:50.992Zfffffe00c6258460 mac:mac_provider_tx+6d ()
6572026-03-31T22:14:51.037Zfffffe00c6258520 mac:mac_tx+28b ()
6582026-03-31T22:14:51.083Zfffffe00c6258570 dld:str_mdata_fastpath_put+8e ()
6592026-03-31T22:14:51.128Zfffffe00c6258680 ip:ip_xmit+9ce ()
6602026-03-31T22:14:51.172Zfffffe00c62588d0 ip:ire_send_wire_v4+3c2 ()
6612026-03-31T22:14:51.218Zfffffe00c6258950 ip:conn_ip_output+18f ()
6622026-03-31T22:14:51.264Zfffffe00c6258a30 ip:udp_output_connected+131 ()
6632026-03-31T22:14:51.340Zfffffe00c6258ad0 ip:udp_send+1bf ()
6642026-03-31T22:14:51.386Zfffffe00c6258b60 sockfs:so_sendmsg+201 ()
6652026-03-31T22:14:51.430Zfffffe00c6258bc0 sockfs:socket_sendmsg+65 ()
6662026-03-31T22:14:51.463Zfffffe00c6258c60 sockfs:sendit+186 ()
6672026-03-31T22:14:51.508Zfffffe00c6258f00 sockfs:sendmsg+2c6 ()
6682026-03-31T22:14:51.549Zfffffe00c6258f10 unix:brand_sys_syscall+1fe ()
6692026-03-31T22:14:51.594Zskipping system dump - no dump device configured
6702026-03-31T22:14:51.630Zrebooting...
6712026-03-31T22:15:38.568Z^[[2J^[[1;1H^[[8;25;80t^[[2J^[[1;1H^[[1;1H^[[0;37;40m^[[2J^[[1;1H^[[25;
6722026-03-31T22:15:39.506Z8H98^[[1;1H^[[25;78H9D^[[1;1H^[[25;78H9A^[[1;1H^[[25;78H9C^[[1;1H^[[25;78
6732026-03-31T22:15:39.546Z9D^[[1;1H^[[25;78H9A^[[1;1H^[[25;78H9C^[[1;1H^[[25;78HB4^[[1;1H^[[25;78HB
6742026-03-31T22:15:42.177Z^[[1;1H^[[25;78HB4^[[1;1H^[[1;1H^[[1;1HPress <DEL> to enter setup, <F1
6752026-03-31T22:15:43.950Z> Display Boot Menu, <F12> Force Network Boot ^[[25;78H92^[[2;1H^[
6762026-03-31T22:15:43.985Z25;78HA0^[[2;1H^[[25;78HA0^[[2;1H^[[25;78H99^[[2;1H^[[25;78H92^[[2;1H^[[2
6772026-03-31T22:15:44.026Z;78H92^[[2;1H^[[25;78H72^[[2;1H^[[25;78HA0^[[2;1H^[[25;78HA2^[[2;1H^[[25;
6782026-03-31T22:15:44.059Z8HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78
6792026-03-31T22:15:44.450ZA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78HA2^[[2;1H^[[25;78H92^[[2;1H^[[25;
6802026-03-31T22:15:44.506Z8HB2^[[2;1H^[[2J^[[1;1H^[[01;00H^[[0m^[[2;37;40mInitializing Intel(R) B
6812026-03-31T22:15:44.584Zot Agent GE v1.5.53 ^[[02;00HP
6822026-03-31T22:15:44.620ZE 2.1 Build 092 (WfM 2.0) ^[
6832026-03-31T22:15:44.651Z03;00HPress Ctrl+S to enter the Setup Menu. ^[
6842026-03-31T22:15:44.685Z04;00H ^[
6852026-03-31T22:15:44.750Z05;00H ^[
6862026-03-31T22:15:44.797Z06;00H ^[
6872026-03-31T22:15:44.842Z07;00H ^[
6882026-03-31T22:15:44.887Z08;00H ^[
6892026-03-31T22:15:44.932Z09;00H ^[
6902026-03-31T22:15:45.010Z10;00H ^[
6912026-03-31T22:15:45.044Z11;00H ^[
6922026-03-31T22:15:45.077Z12;00H ^[
6932026-03-31T22:15:45.111Z13;00H ^[
6942026-03-31T22:15:45.145Z14;00H ^[
6952026-03-31T22:15:45.178Z15;00H ^[
6962026-03-31T22:15:45.511Z16;00H ^[
6972026-03-31T22:15:45.558Z17;00H ^[
6982026-03-31T22:15:45.625Z18;00H ^[
6992026-03-31T22:15:45.682Z19;00H ^[
7002026-03-31T22:15:45.715Z20;00H ^[
7012026-03-31T22:15:45.760Z21;00H ^[
7022026-03-31T22:15:45.793Z22;00H ^[
7032026-03-31T22:15:45.828Z23;00H ^[
7042026-03-31T22:15:45.861Z24;00H ^[
7052026-03-31T22:15:45.895Z24;00H^[[03;39H^[[03;00HPress Ctrl+S to enter the Setup Menu..
7062026-03-31T22:15:46.508Z^[[03;39H^[[01;00HInitializ
7072026-03-31T22:15:46.540Zng Intel(R) Boot Agent GE v1.5.53 ^[
7082026-03-31T22:15:46.889Z02;00HPXE 2.1 Build 092 (WfM 2.0) ^[
7092026-03-31T22:15:46.940Z03;00H ^[
7102026-03-31T22:15:46.996Z04;00H ^[
7112026-03-31T22:15:47.199Z05;00HInitializing Intel(R) Boot Agent GE v1.5.53 ^[
7122026-03-31T22:15:47.232Z06;00HPXE 2.1 Build 092 (WfM 2.0) ^[
7132026-03-31T22:15:47.266Z07;00HPress Ctrl+S to enter the Setup Menu. ^[
7142026-03-31T22:15:47.299Z08;00H ^[
7152026-03-31T22:15:47.333Z09;00H ^[
7162026-03-31T22:15:47.422Z10;00H ^[
7172026-03-31T22:15:47.794Z11;00H ^[
7182026-03-31T22:15:47.968Z12;00H ^[
7192026-03-31T22:15:48.002Z13;00H ^[
7202026-03-31T22:15:48.047Z14;00H ^[
7212026-03-31T22:15:48.092Z15;00H ^[
7222026-03-31T22:15:48.158Z16;00H ^[
7232026-03-31T22:15:48.204Z17;00H ^[
7242026-03-31T22:15:48.249Z18;00H ^[
7252026-03-31T22:15:48.293Z19;00H ^[
7262026-03-31T22:15:48.347Z20;00H ^[
7272026-03-31T22:15:48.382Z21;00H ^[
7282026-03-31T22:15:48.426Z22;00H ^[
7292026-03-31T22:15:48.515Z23;00H ^[
7302026-03-31T22:15:48.560Z24;00H ^[
7312026-03-31T22:15:48.593Z24;00H^[[07;39H^[[07;00HPress Ctrl+S to enter the Setup Menu..
7322026-03-31T22:15:51.263Z^[[07;39H^[[2J^[[1;1H^[[8;25;80t^[
7332026-03-31T22:15:51.306Z2J^[[1;1H^[[1;1H^[[0;37;40mPress <DEL> to enter setup, <F10> Display
7342026-03-31T22:15:51.341ZBoot Menu, <F12> Force Network Boot B2^[
7352026-03-31T22:15:51.674Z2;1H^[[2;1H^[[2;1H
7362026-03-31T22:15:51.761Z^[[3;1H^[[3;1HBIOS Version: MZ32-AR0-
7372026-03-31T22:15:51.941Z0-M06 ^[[4;1H^[[4
7382026-03-31T22:15:52.001Z1HBIOS Date: 07/10/2021 ^[
7392026-03-31T22:15:52.048Z5;1H^[[5;1HProcessor Type: AMD EPYC 7413 24-Core Processor
7402026-03-31T22:15:52.097Z^[[6;1H^[[6;1HProcessor Speed: 2650 MHz
7412026-03-31T22:15:52.408Z^[[7;1H^[[7;1HTot
7422026-03-31T22:15:52.530Zl Memory: 131072MB (DDR4 3200) ^[
7432026-03-31T22:15:52.791Z8;1H^[[8;1HBMC Firmware Version: 12.54.05
7442026-03-31T22:15:53.078Z^[[9;1H^[[9;1HBMC IP : 172.20.4.56
7452026-03-31T22:15:53.125Z^[[10;1H
7462026-03-31T22:15:53.168Z^[
7472026-03-31T22:15:53.227Z11;1H^[[11;1H
7482026-03-31T22:15:53.268Z^[[12;1H^[[12;1HUSB Devices total: 0 KBDs
7492026-03-31T22:15:53.303Z0 MICE, 0 MASS, 3 HUBs ^[[13;1H^[[13
7502026-03-31T22:15:53.337Z1H ^[
7512026-03-31T22:15:53.436Z14;1H^[[14;1HDetected ATA/ATAPI Devices...
7522026-03-31T22:15:53.470Z^[[15;1H^[[15;1H
7532026-03-31T22:15:53.504Z^[[16;1H^[[16
7542026-03-31T22:15:53.539Z1HDetected NVME Devices... ^[
7552026-03-31T22:15:53.572Z17;1H^[[17;1HHP SSD EX950 1TB Size: 1024.2GB
7562026-03-31T22:15:53.605Z^[[18;1H^[[18;1HHP SSD EX950 1TB Size: 10
7572026-03-31T22:15:55.972Z4.2GB ^[[19;1H^[[19
7582026-03-31T22:15:56.576Z1HPreparing Bios Setup configuration.... ^[
7592026-03-31T22:15:56.605Z20;1H^[[20;1HImporting Bios settings from BMC.
7602026-03-31T22:15:57.505Z^[[21;1H^[[21;1H
7612026-03-31T22:16:11.143Z^[[2J^[[1;1H^[
7622026-03-31T22:16:12.348Z2J^[[1;1H^[[2J^[[1;1H^[[01;00H
7632026-03-31T22:16:29.000ZiPXE initialising devices...ok
7642026-03-31T22:16:29.042Z^[[0m
7652026-03-31T22:16:29.090Z^[[1miPXE 1.21.1+ (g8bff5)^[[0m -- Open Source Network Boot Firmwar
7662026-03-31T22:16:29.134Z-- ^[[36mhttp://ipxe.org^[[0m
7672026-03-31T22:16:29.167ZFeatures: DNS HTTP iSCSI TFTP SRP AoE ELF MBOOT PXE bzImage Menu PXEXT
7682026-03-31T22:16:31.234Znet0: 18:c0:4d:0d:a0:2a using i350 on 0000:c3:00.0 (Ethernet) [open]
7692026-03-31T22:16:31.268Z[Link:up, TX:0 TXE:1 RX:0 RXE:0]
7702026-03-31T22:16:31.320Z[TXE: 1 x "Network unreachable (http://ipxe.org/28086011)"]
7712026-03-31T22:16:34.294ZConfiguring (net0 18:c0:4d:0d:a0:2a)...... ok
7722026-03-31T22:16:34.361Znet0: 10.151.2.100/255.255.255.0 gw 10.151.2.1
7732026-03-31T22:16:34.417ZFilename: http://172.20.0.5:9969/ipxe/sock
7742026-03-31T22:16:34.455Zhttp://172.20.0.5:9969/ipxe/sock... ok
7752026-03-31T22:16:34.512Zsock : 555 bytes [script]
7762026-03-31T22:18:37.583Zworker failed without completing job
7772026-03-31T22:18:37.585Ztask 2 was incomplete, marked failed