01KM36PKXVFZVKRDW3M0PTM3QK: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.39",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KM36QRFZGXMR5EADZN956973

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-03-19T14:30:32.242Zjob dependencies complete; ready to run (waiting for 23 m 25 s)
22026-03-19T14:31:39.137Zjob assigned to worker 01KM382V6346F8R7MYT9APYGPR [factory lab, sock/16374] (queued for 1 m 6 s)
32026-03-19T14:31:39.169Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-03-19T14:31:39.213Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52026-03-19T14:31:39.252Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62026-03-19T14:31:39.288Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72026-03-19T14:31:39.325Zhttp://172.20.0.5:9969/postboot/sock... ok
82026-03-19T14:31:39.359ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92026-03-19T14:31:39.392ZWARNING: cpu0: failed to update microcode from version 0xa001133 to 0xa0011de
102026-03-19T14:31:39.425ZWARNING: cpu0: using older fallback microcode; update the system firmware
112026-03-19T14:31:39.459Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d5
122026-03-19T14:31:39.492ZOxide Helios Version helios-2.0.23743 64-bit
132026-03-19T14:31:39.526ZConfiguring devices.
142026-03-19T14:31:39.560Z##################################################
152026-03-19T14:31:39.594Z#### ############# ##
162026-03-19T14:31:39.627Z### ### ############ ##
172026-03-19T14:31:39.661Z## ### # ## ### ## Oxide Computer Company ##
182026-03-19T14:31:39.694Z## ## ## ### # ### ##
192026-03-19T14:31:39.727Z## # ### #### #### This Station Under ##
202026-03-19T14:31:39.800Z### ### #### # ### Computer Control ##
212026-03-19T14:31:39.834Z#### #### ### ## ##
222026-03-19T14:31:39.866Z##################################################
232026-03-19T14:31:39.899ZHostname: sock
242026-03-19T14:31:39.933ZLoading smf(7) service descriptions:
252026-03-19T14:31:40.792Zdownloading input: /input/package/work/package.tar.gz
262026-03-19T14:38:28.270Zdownloaded input: /input/package/work/package.tar.gz
 
272026-03-19T14:38:28.270Zstarting task 0: "setup"
282026-03-19T14:38:28.297Z++ uname -s
292026-03-19T14:38:28.297Z+ kern=SunOS
302026-03-19T14:38:28.297Z+ build_user=build
312026-03-19T14:38:28.297Z+ build_uid=12345
322026-03-19T14:38:28.297Z+ work_dir=/work
332026-03-19T14:38:28.297Z+ input_dir=/input
342026-03-19T14:38:28.297Z+ [[ 0 == 12345 ]]
352026-03-19T14:38:28.297Z+ case "$kern" in
362026-03-19T14:38:28.297Z+ groupadd -g 12345 build
372026-03-19T14:38:28.297Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-03-19T14:38:30.285Z+ zfs create -o mountpoint=/work rpool/work
392026-03-19T14:38:30.310Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-03-19T14:38:30.310Z+ home_fs=zfs
412026-03-19T14:38:30.310Z+ [[ zfs == autofs ]]
422026-03-19T14:38:30.310Z+ mkdir -p /home/build
432026-03-19T14:38:30.334Z+ chown build:build /home/build /work
442026-03-19T14:38:32.313Z+ chmod 0700 /home/build /work
452026-03-19T14:38:32.338Zprocess exited: duration 4051 ms, exit code 0
 
462026-03-19T14:38:32.387Zstarting task 1: "authentication"
472026-03-19T14:38:32.433Zprocess exited: duration 45 ms, exit code 0
 
482026-03-19T14:38:32.482Zstarting task 2: "build"
492026-03-19T14:38:32.507Z+ trap _exit_trap EXIT
502026-03-19T14:38:32.507Z+ OPTE_COMMIT=
512026-03-19T14:38:32.507Z+ [[ x != \x ]]
522026-03-19T14:38:32.507Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
532026-03-19T14:38:32.601Zsecpolicy_fs_linkdir: 0x55 = 0x55
542026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
552026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
562026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+3: 0 = 0xe5
572026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+4: 0 = 0xb8
582026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+5: 0 = 0x1
592026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
602026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
612026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
622026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
632026-03-19T14:38:32.601Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
642026-03-19T14:38:32.626Z+ [[ -d /opt/oxide ]]
652026-03-19T14:38:32.626Z+ pfexec mv /opt/oxide /opt/oxide-underneath
662026-03-19T14:38:32.626Z+ pfexec mkdir /opt/oxide
672026-03-19T14:38:32.626Z+ pfexec mount -F tmpfs -O swap /opt/oxide
682026-03-19T14:38:32.626Z+ [[ -d /opt/oxide-underneath ]]
692026-03-19T14:38:32.626Z+ cd /opt/oxide-underneath
702026-03-19T14:38:32.626Z+ pfexec tar ceEp@/f - .
712026-03-19T14:38:32.626Z+ cd /opt/oxide
722026-03-19T14:38:32.626Z+ pfexec tar xveEp@/f -
732026-03-19T14:38:32.669Zx ., 0 bytes, 0 tape blocks
742026-03-19T14:38:32.669Zx ./opte, 0 bytes, 0 tape blocks
752026-03-19T14:38:32.670Zx ./opte/bin, 0 bytes, 0 tape blocks
762026-03-19T14:38:32.670Zx ./opte/bin/opteadm, 22792392 bytes, 44517 tape blocks
772026-03-19T14:38:32.694Z+ pfexec rm -rf /opt/oxide-underneath
782026-03-19T14:38:32.694Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
792026-03-19T14:38:32.719Z+ pfexec mkdir /opt/oxide/work
802026-03-19T14:38:32.719Z+ pfexec chown build:build /opt/oxide/work
812026-03-19T14:38:32.719Z+ cd /opt/oxide/work
822026-03-19T14:38:32.719Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
832026-03-19T14:38:53.422ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
842026-03-19T14:38:53.422Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
852026-03-19T14:38:53.422Zx out/target/test, 107 bytes, 1 tape blocks
862026-03-19T14:38:53.422Zx out/npuzone/npuzone, 28059200 bytes, 54804 tape blocks
872026-03-19T14:38:53.422Zx package-manifest.toml, 34895 bytes, 69 tape blocks
882026-03-19T14:38:53.422Zx smf/sled-agent/non-gimlet/config.toml, 5328 bytes, 11 tape blocks
892026-03-19T14:38:53.422Zx target/release/omicron-package, 41533528 bytes, 81121 tape blocks
902026-03-19T14:38:53.422Zx target/release/xtask, 11707104 bytes, 22866 tape blocks
912026-03-19T14:38:53.422Zx target/debug/bootstrap, 344766080 bytes, 673372 tape blocks
922026-03-19T14:38:53.422Zx tests/bootstrap-73543619dd1e712a, 32061232 bytes, 62620 tape blocks
932026-03-19T14:38:53.422Zx tests/commtest-71f154827dfcabb2, 33172184 bytes, 64790 tape blocks
942026-03-19T14:38:53.422Zx tests/dhcp_server-a47ac48134428f42, 32061248 bytes, 62620 tape blocks
952026-03-19T14:38:53.422Zx tests/end_to_end_tests-e1218ca0fcbd2f2d, 388021648 bytes, 757855 tape blocks
962026-03-19T14:38:53.422Zx out/clickhouse.tar.gz, 308949476 bytes, 603417 tape blocks
972026-03-19T14:38:53.422Zx out/clickhouse_keeper.tar.gz, 294065204 bytes, 574347 tape blocks
982026-03-19T14:38:53.422Zx out/clickhouse_server.tar.gz, 308948535 bytes, 603416 tape blocks
992026-03-19T14:38:53.422Zx out/cockroachdb.tar.gz, 160290023 bytes, 313067 tape blocks
1002026-03-19T14:38:53.422Zx out/crucible-pantry-zone.tar.gz, 39925578 bytes, 77980 tape blocks
1012026-03-19T14:38:53.422Zx out/crucible-utils.tar, 18169344 bytes, 35487 tape blocks
1022026-03-19T14:38:53.422Zx out/crucible-zone.tar.gz, 54403054 bytes, 106256 tape blocks
1032026-03-19T14:38:53.422Zx out/external-dns.tar.gz, 44650589 bytes, 87209 tape blocks
1042026-03-19T14:38:53.423Zx out/internal-dns.tar.gz, 44650185 bytes, 87208 tape blocks
1052026-03-19T14:38:53.423Zx out/mg-ddm-gz.tar, 80258048 bytes, 156754 tape blocks
1062026-03-19T14:38:53.423Zx out/nexus.tar.gz, 214586276 bytes, 419114 tape blocks
1072026-03-19T14:38:53.423Zx out/ntp.tar.gz, 26028102 bytes, 50837 tape blocks
1082026-03-19T14:38:53.423Zx out/omicron-gateway-softnpu.tar.gz, 28973406 bytes, 56589 tape blocks
1092026-03-19T14:38:53.423Zx out/omicron-sled-agent.tar, 190887936 bytes, 372828 tape blocks
1102026-03-19T14:38:53.423Zx out/overlay.tar.gz, 3030 bytes, 6 tape blocks
1112026-03-19T14:38:53.423Zx out/oximeter.tar.gz, 58843921 bytes, 114930 tape blocks
1122026-03-19T14:38:53.423Zx out/oxlog.tar, 3872256 bytes, 7563 tape blocks
1132026-03-19T14:38:53.423Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1142026-03-19T14:38:53.423Zx out/propolis-server.tar.gz, 76424187 bytes, 149266 tape blocks
1152026-03-19T14:38:53.423Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1162026-03-19T14:38:53.423Zx out/switch-softnpu.tar.gz, 376969264 bytes, 736269 tape blocks
1172026-03-19T14:38:53.447Z
1182026-03-19T14:38:53.447Zreal 20.695230330
1192026-03-19T14:38:53.447Zuser 18.561125393
1202026-03-19T14:38:53.447Zsys 3.657030586
1212026-03-19T14:38:53.447Ztrap 0.000812963
1222026-03-19T14:38:53.447Ztflt 0.000000000
1232026-03-19T14:38:53.448Zdflt 0.000000000
1242026-03-19T14:38:53.448Zkflt 0.000000000
1252026-03-19T14:38:53.448Zlock 0.000000000
1262026-03-19T14:38:53.448Zslp 18.643855310
1272026-03-19T14:38:53.448Zlat 0.525128484
1282026-03-19T14:38:53.448Zstop 0.000135561
1292026-03-19T14:38:53.448Z+ source .github/buildomat/ci-env.sh
1302026-03-19T14:38:53.448Z++ export CARGO_TERM_COLOR=always
1312026-03-19T14:38:53.448Z++ CARGO_TERM_COLOR=always
1322026-03-19T14:38:53.448Z+ bmat address ls
1332026-03-19T14:38:53.448ZNAME CIDR FIRST LAST COUNT
1342026-03-19T14:38:53.448Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1352026-03-19T14:38:53.448Z++ bmat address ls -f extra -Ho first,last,count
1362026-03-19T14:38:53.448Z+ set -- 10.151.2.164 10.151.2.227 64
1372026-03-19T14:38:53.448Z+ EXTRA_IP_START=10.151.2.164
1382026-03-19T14:38:53.448Z+ EXTRA_IP_END=10.151.2.227
1392026-03-19T14:38:53.448Z+ EXTRA_IP_COUNT=64
1402026-03-19T14:38:53.448Z+ (( EXTRA_IP_COUNT >= 32 ))
1412026-03-19T14:38:53.448Z+ EXTRA_IP_BASE=10.151.2
1422026-03-19T14:38:53.448Z+ EXTRA_IP_FOCTET=164
1432026-03-19T14:38:53.448Z+ EXTRA_IP_LOCTET=227
1442026-03-19T14:38:53.448Z+ SERVICE_IP_POOL_START=10.151.2.164
1452026-03-19T14:38:53.448Z+ SERVICE_IP_POOL_END=10.151.2.173
1462026-03-19T14:38:53.448Z+ DNS_IP1=10.151.2.164
1472026-03-19T14:38:53.448Z+ DNS_IP2=10.151.2.165
1482026-03-19T14:38:53.448Z+ UPLINK_IP=10.151.2.174
1492026-03-19T14:38:53.448Z+ IPPOOL_START=10.151.2.175
1502026-03-19T14:38:53.448Z+ IPPOOL_END=10.151.2.227
1512026-03-19T14:38:53.448Z++ ipadm show-addr -po type,addr
1522026-03-19T14:38:53.448Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1532026-03-19T14:38:53.448Z+ GATEWAY_IP=10.151.2.100
1542026-03-19T14:38:53.448Z+ [[ -n 10.151.2.100 ]]
1552026-03-19T14:38:53.448Z+ ping -s 10.151.2.100 56 1
1562026-03-19T14:38:53.448ZPING 10.151.2.100: 56 data bytes
1572026-03-19T14:38:53.448Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.012 ms
1582026-03-19T14:38:53.448Z
1592026-03-19T14:38:53.448Z----10.151.2.100 PING Statistics----
1602026-03-19T14:38:53.448Z1 packets transmitted, 1 packets received, 0% packet loss
1612026-03-19T14:38:53.448Zround-trip (ms) min/avg/max/stddev = 0.012/0.012/0.012/-nan
1622026-03-19T14:38:53.449Z++ arp -an
1632026-03-19T14:38:53.449Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1642026-03-19T14:38:53.473Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1652026-03-19T14:38:53.473Z+ routeadm -e ipv4-forwarding -u
1662026-03-19T14:38:53.545Z+ PXA_START=10.151.2.164
1672026-03-19T14:38:53.545Z+ PXA_END=10.151.2.227
1682026-03-19T14:38:53.545Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
1692026-03-19T14:38:53.728Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1702026-03-19T14:38:53.753ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1712026-03-19T14:38:53.803Zcreating virtual hardware
1722026-03-19T14:38:53.827Zcreating /scratch/m2_0.vdev
1732026-03-19T14:38:53.827Zcreating /scratch/m2_1.vdev
1742026-03-19T14:38:53.827Zcreating /scratch/u2_0.vdev
1752026-03-19T14:38:53.827Zcreating /scratch/u2_1.vdev
1762026-03-19T14:38:53.827Zcreating /scratch/u2_2.vdev
1772026-03-19T14:38:53.827Zcreating /scratch/u2_3.vdev
1782026-03-19T14:38:53.827Zcreating /scratch/u2_4.vdev
1792026-03-19T14:38:53.827Zcreating /scratch/u2_5.vdev
1802026-03-19T14:38:53.827Zcreating /scratch/u2_6.vdev
1812026-03-19T14:38:53.827Zcreating /scratch/u2_7.vdev
1822026-03-19T14:38:53.827Zcreating /scratch/u2_8.vdev
1832026-03-19T14:38:53.851ZSimnet net0/sc0_0 exists
1842026-03-19T14:38:53.876ZSimnet net1/sc1_0 exists
1852026-03-19T14:38:53.876ZVnic sc0_1 exists
1862026-03-19T14:39:06.798ZUsing 10.151.2.100 as gateway ip
1872026-03-19T14:39:06.798Zusing 18:C0:4D:0D:A0:2A as gateway mac
1882026-03-19T14:39:06.798Zconfiguring SoftNPU ARP entry
1892026-03-19T14:39:06.825Zconfiguring SoftNPU proxy ARP
1902026-03-19T14:39:06.876ZSoftNPU state:
1912026-03-19T14:39:06.877Z local v6:
1922026-03-19T14:39:06.877Z local v4:
1932026-03-19T14:39:06.877Z router v6_idx:
1942026-03-19T14:39:06.877Z router v6_routes:
1952026-03-19T14:39:06.877Z router v4_idx:
1962026-03-19T14:39:06.877Z router v4_routes:
1972026-03-19T14:39:06.877Z resolver v4:
1982026-03-19T14:39:06.877Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
1992026-03-19T14:39:06.877Z resolver v6:
2002026-03-19T14:39:06.877Z nat v4:
2012026-03-19T14:39:06.877Z nat_v6:
2022026-03-19T14:39:06.877Z attached subnet v4:
2032026-03-19T14:39:06.877Z attached subnet v6:
2042026-03-19T14:39:06.877Z port_mac:
2052026-03-19T14:39:06.877Z proxy arp:
2062026-03-19T14:39:06.877Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2072026-03-19T14:39:06.877Zcreated virtual hardware
2082026-03-19T14:39:06.902Z
2092026-03-19T14:39:06.902Zreal 13.145204593
2102026-03-19T14:39:06.902Zuser 0.653825281
2112026-03-19T14:39:06.902Zsys 3.409153126
2122026-03-19T14:39:06.902Ztrap 0.000093287
2132026-03-19T14:39:06.902Ztflt 0.000215797
2142026-03-19T14:39:06.902Zdflt 0.000518877
2152026-03-19T14:39:06.902Zkflt 0.000007930
2162026-03-19T14:39:06.902Zlock 10:11.012834904
2172026-03-19T14:39:06.902Zslp 39.712647482
2182026-03-19T14:39:06.902Zlat 0.134630890
2192026-03-19T14:39:06.902Zstop 0.247144445
2202026-03-19T14:39:06.902Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2212026-03-19T14:39:06.928Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2222026-03-19T14:39:06.974Z+ diff -u pkg/config.toml~ pkg/config.toml
2232026-03-19T14:39:06.975Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2242026-03-19T14:39:06.975Z+++ pkg/config.toml Thu Mar 19 14:39:06 2026
2252026-03-19T14:39:06.975Z@@ -29,18 +29,18 @@
2262026-03-19T14:39:06.975Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2272026-03-19T14:39:06.975Z # followed by a string that is embedded into their fake serial values.
2282026-03-19T14:39:06.975Z vdevs = [
2292026-03-19T14:39:06.975Z- "m2_0.vdev",
2302026-03-19T14:39:06.975Z- "m2_1.vdev",
2312026-03-19T14:39:06.975Z+ "/scratch/m2_0.vdev",
2322026-03-19T14:39:06.975Z+ "/scratch/m2_1.vdev",
2332026-03-19T14:39:06.975Z
2342026-03-19T14:39:06.975Z- "u2_0.vdev",
2352026-03-19T14:39:06.975Z- "u2_1.vdev",
2362026-03-19T14:39:06.975Z- "u2_2.vdev",
2372026-03-19T14:39:06.975Z- "u2_3.vdev",
2382026-03-19T14:39:06.975Z- "u2_4.vdev",
2392026-03-19T14:39:06.975Z- "u2_5.vdev",
2402026-03-19T14:39:06.975Z- "u2_6.vdev",
2412026-03-19T14:39:06.975Z- "u2_7.vdev",
2422026-03-19T14:39:06.975Z- "u2_8.vdev",
2432026-03-19T14:39:06.975Z+ "/scratch/u2_0.vdev",
2442026-03-19T14:39:06.975Z+ "/scratch/u2_1.vdev",
2452026-03-19T14:39:06.975Z+ "/scratch/u2_2.vdev",
2462026-03-19T14:39:06.975Z+ "/scratch/u2_3.vdev",
2472026-03-19T14:39:06.975Z+ "/scratch/u2_4.vdev",
2482026-03-19T14:39:06.975Z+ "/scratch/u2_5.vdev",
2492026-03-19T14:39:06.975Z+ "/scratch/u2_6.vdev",
2502026-03-19T14:39:06.975Z+ "/scratch/u2_7.vdev",
2512026-03-19T14:39:06.975Z+ "/scratch/u2_8.vdev",
2522026-03-19T14:39:06.975Z ]
2532026-03-19T14:39:06.975Z
2542026-03-19T14:39:06.975Z # The amount of memory held back for services which exist between zero and one
2552026-03-19T14:39:06.975Z+ true
2562026-03-19T14:39:06.975Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2572026-03-19T14:39:06.975Z+ EXPECTED_ZPOOL_COUNT=9
2582026-03-19T14:39:06.975Z+ echo 'expected number of zpools is 9'
2592026-03-19T14:39:06.976Zexpected number of zpools is 9
2602026-03-19T14:39:06.976Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2612026-03-19T14:39:06.976Z+ SILO_NAME=recovery
2622026-03-19T14:39:06.976Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2632026-03-19T14:39:06.976Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2642026-03-19T14:39:06.976Z+ sed -i~ '
2652026-03-19T14:39:06.976Z /^external_dns_ips/c\
2662026-03-19T14:39:06.976Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2672026-03-19T14:39:06.976Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2682026-03-19T14:39:06.976Z /^first/c\
2692026-03-19T14:39:06.976Zfirst = "10.151.2.164"
2702026-03-19T14:39:06.976Z /^last/c\
2712026-03-19T14:39:06.977Zlast = "10.151.2.173"
2722026-03-19T14:39:06.977Z }
2732026-03-19T14:39:06.977Z /^infra_ip_first/c\
2742026-03-19T14:39:06.977Zinfra_ip_first = "10.151.2.174"
2752026-03-19T14:39:06.977Z /^infra_ip_last/c\
2762026-03-19T14:39:06.977Zinfra_ip_last = "10.151.2.174"
2772026-03-19T14:39:06.977Z /^\[\[rack_network_config.ports/,/^$/ {
2782026-03-19T14:39:06.977Z /^routes/c\
2792026-03-19T14:39:06.977Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2802026-03-19T14:39:06.977Z /^addresses/c\
2812026-03-19T14:39:06.977Zaddresses = \[{address = "10.151.2.174/24"} \]
2822026-03-19T14:39:06.977Z }
2832026-03-19T14:39:06.977Z' pkg/config-rss.toml
2842026-03-19T14:39:06.977Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2852026-03-19T14:39:06.977Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2862026-03-19T14:39:06.977Z+++ pkg/config-rss.toml Thu Mar 19 14:39:06 2026
2872026-03-19T14:39:06.977Z@@ -33,7 +33,7 @@
2882026-03-19T14:39:06.977Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2892026-03-19T14:39:06.977Z # must be contained in one of the "internal services" IP Pool ranges listed
2902026-03-19T14:39:06.977Z # below.
2912026-03-19T14:39:06.977Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2922026-03-19T14:39:06.977Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2932026-03-19T14:39:06.977Z
2942026-03-19T14:39:06.977Z # Initial TLS certificates for the external API
2952026-03-19T14:39:06.977Z #
2962026-03-19T14:39:06.977Z@@ -69,8 +69,8 @@
2972026-03-19T14:39:06.977Z #
2982026-03-19T14:39:06.977Z # For more on this and what to put here, see docs/how-to-run.adoc.
2992026-03-19T14:39:06.977Z [[internal_services_ip_pool_ranges]]
3002026-03-19T14:39:06.977Z-first = "192.168.1.20"
3012026-03-19T14:39:06.977Z-last = "192.168.1.29"
3022026-03-19T14:39:06.977Z+first = "10.151.2.164"
3032026-03-19T14:39:06.977Z+last = "10.151.2.173"
3042026-03-19T14:39:06.977Z
3052026-03-19T14:39:06.977Z # TODO - this configuration is subject to change going forward. Ultimately these
3062026-03-19T14:39:06.977Z # parameters should be provided to the control plane via wicket, but we need to
3072026-03-19T14:39:06.977Z@@ -91,8 +91,8 @@
3082026-03-19T14:39:06.977Z # A range of IP addresses used by Boundary Services on the external network. In
3092026-03-19T14:39:06.977Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3102026-03-19T14:39:06.978Z # With softnpu, only one address is used.
3112026-03-19T14:39:06.978Z-infra_ip_first = "192.168.1.30"
3122026-03-19T14:39:06.978Z-infra_ip_last = "192.168.1.30"
3132026-03-19T14:39:06.978Z+infra_ip_first = "10.151.2.174"
3142026-03-19T14:39:06.978Z+infra_ip_last = "10.151.2.174"
3152026-03-19T14:39:06.978Z
3162026-03-19T14:39:06.978Z # Configurations for BGP routers to run on the scrimlets.
3172026-03-19T14:39:06.978Z bgp = []
3182026-03-19T14:39:06.978Z@@ -100,9 +100,9 @@
3192026-03-19T14:39:06.978Z # You can configure multiple uplinks by repeating the following stanza
3202026-03-19T14:39:06.978Z [[rack_network_config.ports]]
3212026-03-19T14:39:06.978Z # Routes associated with this port.
3222026-03-19T14:39:07.003Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3232026-03-19T14:39:07.003Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3242026-03-19T14:39:07.003Z # Addresses associated with this port.
3252026-03-19T14:39:07.003Z-addresses = [{address = {type = "address", ip_net = "192.168.1.30/24"}}]
3262026-03-19T14:39:07.003Z+addresses = [{address = "10.151.2.174/24"} ]
3272026-03-19T14:39:07.003Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3282026-03-19T14:39:07.003Z port = "qsfp0"
3292026-03-19T14:39:07.003Z # The speed of this port.
3302026-03-19T14:39:07.003Z+ true
3312026-03-19T14:39:07.003Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3322026-03-19T14:39:07.003Za pkg/config-rss.toml 8K
3332026-03-19T14:39:07.003Za pkg/config.toml 6K
3342026-03-19T14:39:07.003Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3352026-03-19T14:39:07.003Z+ TLS_NAME=recovery.sys.oxide.test
3362026-03-19T14:39:07.003Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3372026-03-19T14:39:07.271Z.....+.....+.............+..+.+.....+................+........+............+.+.....+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+.+......+..+.+.....+......+.+.....+....+...+..+.+..+...+..........+...+............+...........................+...+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+....+..+....+.....+......+....+........+.+..+...+.+...............+.......................+...+...+....................................+...+...+...+....+............+........+.............+.....+.......+...+.....+.............+..+...+......+.+.....+....+...+........+...+.+........+...+.........+.+........+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3382026-03-19T14:39:07.939Z.....+..+....+...+......+......+...+...+........+.+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+..........+...+......+.........+..............+.+..+...+...............+......+.............+...+..+..................+...................+.....+.........+......+.+..+......+....+..+.+.....+..........+...+........................+.....+.......+.........+......+.....+.+....................+....+........+...+.......+..+...................+..+.......+..+..........+......+...+......+......+.................+...+....+........+.........................+..+.+.....+....+........+.+.....+....+......+..+..........+..............+...+.+..............+.......+...+..+...............+...+.......+........+.+........................+......+............+........+.......+..+......+.+......+............+...+.....+.......+........+...+...............+....+.................+.+.....+.......+........+...+......+..................+.+...........+..........+...+...........+..........+......+......+........+.+...+..+...+..........+...........+..........+..+......................+........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3392026-03-19T14:39:07.963Z-----
3402026-03-19T14:39:07.987Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3412026-03-19T14:39:07.987Za pkg/initial-tls-cert.pem 2K
3422026-03-19T14:39:07.987Za pkg/initial-tls-key.pem 4K
3432026-03-19T14:39:07.987Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3442026-03-19T14:39:07.987Z+ rmdir pkg
3452026-03-19T14:39:07.987Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3462026-03-19T14:39:07.987Z+ pfexec mkdir -p /usr/oxide
3472026-03-19T14:39:07.987Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3482026-03-19T14:39:08.651Z+ pfexec chmod +x /usr/oxide/oxide
3492026-03-19T14:39:08.675Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3502026-03-19T14:39:26.967Z+ OMICRON_NO_UNINSTALL=1
3512026-03-19T14:39:26.967Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3522026-03-19T14:39:26.991ZLogging to: /opt/oxide/work/out/LOG
3532026-03-19T14:39:27.796Z
3542026-03-19T14:39:27.820Zreal 0.823307123
3552026-03-19T14:39:27.820Zuser 0.167233481
3562026-03-19T14:39:27.821Zsys 4.864319909
3572026-03-19T14:39:27.821Ztrap 0.000012879
3582026-03-19T14:39:27.821Ztflt 0.000195861
3592026-03-19T14:39:27.821Zdflt 0.000183755
3602026-03-19T14:39:27.821Zkflt 0.000000000
3612026-03-19T14:39:27.821Zlock 1:14.131898564
3622026-03-19T14:39:27.821Zslp 1.007586579
3632026-03-19T14:39:27.821Zlat 0.020490624
3642026-03-19T14:39:27.821Zstop 0.073933233
3652026-03-19T14:39:27.821Z+ retry=0
3662026-03-19T14:39:27.821Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3672026-03-19T14:39:27.821Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3682026-03-19T14:39:27.821Z+ [[ 0 -gt 30 ]]
3692026-03-19T14:39:27.821Z+ sleep 1
 
3702026-03-19T14:39:28.100Zsock console login: Mar 19 14:39:27 sock svc.startd[8]: oxide/sled-agent:default failed repeatedly: transitioned to maintenance (see 'svcs -xv' for details)
3712026-03-19T14:39:28.736ZSUNW-MSG-ID: SMF-8000-YX, TYPE: defect, VER: 1, SEVERITY: major
3722026-03-19T14:39:28.793ZEVENT-TIME: Thu Mar 19 14:39:27 UTC 2026
 
3732026-03-19T14:39:28.810Z+ retry=1
3742026-03-19T14:39:28.810Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
 
3752026-03-19T14:39:28.831ZPLATFORM: R152-Z32-00, CSN: GKG8N7212A0003, HOSTNAME: sock
 
3762026-03-19T14:39:28.834Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3772026-03-19T14:39:28.834Z+ [[ 1 -gt 30 ]]
3782026-03-19T14:39:28.834Z+ sleep 1
 
3792026-03-19T14:39:28.875ZSOURCE: software-diagnosis, REV: 0.1
3802026-03-19T14:39:28.920ZEVENT-ID: 99014b36-2abc-4fab-b8ef-d5251a5f7a57
3812026-03-19T14:39:28.964ZDESC: A service failed - the instance is restarting too quickly.
3822026-03-19T14:39:29.028ZRefer to http://illumos.org/msg/SMF-8000-YX for more information.
3832026-03-19T14:39:29.065ZAUTO-RESPONSE: The service has been placed into the maintenance state.
3842026-03-19T14:39:29.099ZIMPACT: svc:/oxide/sled-agent:default is unavailable.
3852026-03-19T14:39:29.132ZREC-ACTION: Run 'svcs -xv svc:/oxide/sled-agent:default' to determine the generic reason why the service failed, the location of any logfiles, and a list of other services impacted.
 
3862026-03-19T14:39:29.824Z+ retry=2
3872026-03-19T14:39:29.824Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3882026-03-19T14:39:29.848Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3892026-03-19T14:39:29.848Z+ [[ 2 -gt 30 ]]
3902026-03-19T14:39:29.848Z+ sleep 1
3912026-03-19T14:39:30.839Z+ retry=3
3922026-03-19T14:39:30.839Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3932026-03-19T14:39:30.863Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3942026-03-19T14:39:30.863Z+ [[ 3 -gt 30 ]]
3952026-03-19T14:39:30.863Z+ sleep 1
3962026-03-19T14:39:31.851Z+ retry=4
3972026-03-19T14:39:31.851Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3982026-03-19T14:39:31.876Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3992026-03-19T14:39:31.876Z+ [[ 4 -gt 30 ]]
4002026-03-19T14:39:31.876Z+ sleep 1
4012026-03-19T14:39:32.863Z+ retry=5
4022026-03-19T14:39:32.863Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4032026-03-19T14:39:32.888Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4042026-03-19T14:39:32.888Z+ [[ 5 -gt 30 ]]
4052026-03-19T14:39:32.888Z+ sleep 1
4062026-03-19T14:39:33.877Z+ retry=6
4072026-03-19T14:39:33.877Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4082026-03-19T14:39:33.902Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4092026-03-19T14:39:33.902Z+ [[ 6 -gt 30 ]]
4102026-03-19T14:39:33.902Z+ sleep 1
4112026-03-19T14:39:34.890Z+ retry=7
4122026-03-19T14:39:34.890Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4132026-03-19T14:39:34.915Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4142026-03-19T14:39:34.915Z+ [[ 7 -gt 30 ]]
4152026-03-19T14:39:34.915Z+ sleep 1
4162026-03-19T14:39:35.903Z+ retry=8
4172026-03-19T14:39:35.903Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4182026-03-19T14:39:35.928Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4192026-03-19T14:39:35.928Z+ [[ 8 -gt 30 ]]
4202026-03-19T14:39:35.928Z+ sleep 1
4212026-03-19T14:39:36.916Z+ retry=9
4222026-03-19T14:39:36.916Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4232026-03-19T14:39:36.941Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4242026-03-19T14:39:36.941Z+ [[ 9 -gt 30 ]]
4252026-03-19T14:39:36.941Z+ sleep 1
4262026-03-19T14:39:37.930Z+ retry=10
4272026-03-19T14:39:37.930Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4282026-03-19T14:39:37.954Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4292026-03-19T14:39:37.954Z+ [[ 10 -gt 30 ]]
4302026-03-19T14:39:37.954Z+ sleep 1
4312026-03-19T14:39:38.942Z+ retry=11
4322026-03-19T14:39:38.942Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4332026-03-19T14:39:38.967Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4342026-03-19T14:39:38.967Z+ [[ 11 -gt 30 ]]
4352026-03-19T14:39:38.967Z+ sleep 1
4362026-03-19T14:39:39.956Z+ retry=12
4372026-03-19T14:39:39.956Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4382026-03-19T14:39:39.980Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4392026-03-19T14:39:39.980Z+ [[ 12 -gt 30 ]]
4402026-03-19T14:39:39.980Z+ sleep 1
4412026-03-19T14:39:40.968Z+ retry=13
4422026-03-19T14:39:40.968Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4432026-03-19T14:39:40.993Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4442026-03-19T14:39:40.993Z+ [[ 13 -gt 30 ]]
4452026-03-19T14:39:40.993Z+ sleep 1
4462026-03-19T14:39:41.981Z+ retry=14
4472026-03-19T14:39:41.981Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4482026-03-19T14:39:42.006Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4492026-03-19T14:39:42.006Z+ [[ 14 -gt 30 ]]
4502026-03-19T14:39:42.006Z+ sleep 1
4512026-03-19T14:39:42.994Z+ retry=15
4522026-03-19T14:39:42.994Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4532026-03-19T14:39:43.019Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4542026-03-19T14:39:43.019Z+ [[ 15 -gt 30 ]]
4552026-03-19T14:39:43.019Z+ sleep 1
4562026-03-19T14:39:44.006Z+ retry=16
4572026-03-19T14:39:44.007Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4582026-03-19T14:39:44.031Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4592026-03-19T14:39:44.031Z+ [[ 16 -gt 30 ]]
4602026-03-19T14:39:44.031Z+ sleep 1
4612026-03-19T14:39:45.020Z+ retry=17
4622026-03-19T14:39:45.020Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4632026-03-19T14:39:45.044Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4642026-03-19T14:39:45.045Z+ [[ 17 -gt 30 ]]
4652026-03-19T14:39:45.045Z+ sleep 1
4662026-03-19T14:39:46.032Z+ retry=18
4672026-03-19T14:39:46.032Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4682026-03-19T14:39:46.056Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4692026-03-19T14:39:46.056Z+ [[ 18 -gt 30 ]]
4702026-03-19T14:39:46.056Z+ sleep 1
4712026-03-19T14:39:47.046Z+ retry=19
4722026-03-19T14:39:47.046Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4732026-03-19T14:39:47.071Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4742026-03-19T14:39:47.071Z+ [[ 19 -gt 30 ]]
4752026-03-19T14:39:47.071Z+ sleep 1
4762026-03-19T14:39:48.059Z+ retry=20
4772026-03-19T14:39:48.060Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4782026-03-19T14:39:48.084Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4792026-03-19T14:39:48.084Z+ [[ 20 -gt 30 ]]
4802026-03-19T14:39:48.084Z+ sleep 1
4812026-03-19T14:39:49.071Z+ retry=21
4822026-03-19T14:39:49.071Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4832026-03-19T14:39:49.096Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4842026-03-19T14:39:49.096Z+ [[ 21 -gt 30 ]]
4852026-03-19T14:39:49.096Z+ sleep 1
4862026-03-19T14:39:50.085Z+ retry=22
4872026-03-19T14:39:50.085Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4882026-03-19T14:39:50.109Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4892026-03-19T14:39:50.109Z+ [[ 22 -gt 30 ]]
4902026-03-19T14:39:50.109Z+ sleep 1
4912026-03-19T14:39:51.098Z+ retry=23
4922026-03-19T14:39:51.098Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4932026-03-19T14:39:51.123Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4942026-03-19T14:39:51.123Z+ [[ 23 -gt 30 ]]
4952026-03-19T14:39:51.123Z+ sleep 1
4962026-03-19T14:39:52.112Z+ retry=24
4972026-03-19T14:39:52.112Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4982026-03-19T14:39:52.137Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4992026-03-19T14:39:52.137Z+ [[ 24 -gt 30 ]]
5002026-03-19T14:39:52.137Z+ sleep 1
5012026-03-19T14:39:53.124Z+ retry=25
5022026-03-19T14:39:53.124Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5032026-03-19T14:39:53.149Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5042026-03-19T14:39:53.149Z+ [[ 25 -gt 30 ]]
5052026-03-19T14:39:53.149Z+ sleep 1
5062026-03-19T14:39:54.137Z+ retry=26
5072026-03-19T14:39:54.137Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5082026-03-19T14:39:54.161Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5092026-03-19T14:39:54.161Z+ [[ 26 -gt 30 ]]
5102026-03-19T14:39:54.161Z+ sleep 1
5112026-03-19T14:39:55.151Z+ retry=27
5122026-03-19T14:39:55.151Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5132026-03-19T14:39:55.175Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5142026-03-19T14:39:55.175Z+ [[ 27 -gt 30 ]]
5152026-03-19T14:39:55.175Z+ sleep 1
5162026-03-19T14:39:56.164Z+ retry=28
5172026-03-19T14:39:56.164Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5182026-03-19T14:39:56.190Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5192026-03-19T14:39:56.190Z+ [[ 28 -gt 30 ]]
5202026-03-19T14:39:56.190Z+ sleep 1
5212026-03-19T14:39:57.177Z+ retry=29
5222026-03-19T14:39:57.177Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5232026-03-19T14:39:57.201Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5242026-03-19T14:39:57.201Z+ [[ 29 -gt 30 ]]
5252026-03-19T14:39:57.201Z+ sleep 1
5262026-03-19T14:39:58.190Z+ retry=30
5272026-03-19T14:39:58.190Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5282026-03-19T14:39:58.214Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5292026-03-19T14:39:58.215Z+ [[ 30 -gt 30 ]]
5302026-03-19T14:39:58.215Z+ sleep 1
5312026-03-19T14:39:59.203Z+ retry=31
5322026-03-19T14:39:59.203Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5332026-03-19T14:39:59.228Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5342026-03-19T14:39:59.228Z+ [[ 31 -gt 30 ]]
5352026-03-19T14:39:59.228Z+ echo 'Failed to reach switch zone after 30 attempts'
5362026-03-19T14:39:59.228Z+ exit 1
5372026-03-19T14:39:59.228ZFailed to reach switch zone after 30 attempts
5382026-03-19T14:39:59.228Z+ _exit_trap
5392026-03-19T14:39:59.228Z+ local status=1
5402026-03-19T14:39:59.228Z+ set +o errexit
5412026-03-19T14:39:59.228Z+ [[ x != \x ]]
5422026-03-19T14:39:59.228Z+ pfexec svcadm -Z disable -s cron
5432026-03-19T14:39:59.269Z+ [[ 1 -eq 0 ]]
5442026-03-19T14:39:59.269Z+ set -o xtrace
5452026-03-19T14:39:59.269Z+ banner evidence
5462026-03-19T14:39:59.293Z
5472026-03-19T14:39:59.293Z ###### # # # ##### ###### # # #### ######
5482026-03-19T14:39:59.293Z # # # # # # # ## # # # #
5492026-03-19T14:39:59.293Z ##### # # # # # ##### # # # # #####
5502026-03-19T14:39:59.293Z # # # # # # # # # # # #
5512026-03-19T14:39:59.293Z # # # # # # # # ## # # #
5522026-03-19T14:39:59.293Z ###### ## # ##### ###### # # #### ######
5532026-03-19T14:39:59.293Z
5542026-03-19T14:39:59.293Z+ zoneadm list -civ
5552026-03-19T14:39:59.293Z ID NAME STATUS PATH BRAND IP
5562026-03-19T14:39:59.293Z 0 global running / ipkg shared
5572026-03-19T14:39:59.294Z 1 sidecar_softnpu running /sidecar/sidecar_softnpu omicron1 excl
5582026-03-19T14:39:59.294Z+ pfexec dladm show-phys -m
5592026-03-19T14:39:59.294ZLINK SLOT ADDRESS INUSE CLIENT
5602026-03-19T14:39:59.294Zigb0 primary 18:c0:4d:d:a0:2a yes igb0
5612026-03-19T14:39:59.294Zigb1 primary 18:c0:4d:d:a0:2b no --
5622026-03-19T14:39:59.294Z+ pfexec dladm show-link
5632026-03-19T14:39:59.294ZLINK CLASS MTU STATE BRIDGE OVER
5642026-03-19T14:39:59.294Zigb0 phys 1500 up -- --
5652026-03-19T14:39:59.294Zigb1 phys 1500 unknown -- --
5662026-03-19T14:39:59.294Znet0 simnet 9000 up -- sc0_0
5672026-03-19T14:39:59.294Zsc0_0 simnet 9000 up -- net0
5682026-03-19T14:39:59.294Znet1 simnet 9000 up -- sc1_0
5692026-03-19T14:39:59.294Zsc1_0 simnet 9000 up -- net1
5702026-03-19T14:39:59.294Zsc0_1 vnic 1500 up -- igb0
5712026-03-19T14:39:59.294Ztfportrear0_0 simnet 9000 up -- sidecar_shi0
5722026-03-19T14:39:59.294Zsidecar_shi0 simnet 9000 up -- tfportrear0_0
5732026-03-19T14:39:59.294Ztfportqsfp0_0 simnet 9000 up -- sidecar_shi1
5742026-03-19T14:39:59.294Zsidecar_shi1 simnet 9000 up -- tfportqsfp0_0
5752026-03-19T14:39:59.294Z+ pfexec dladm show-vnic
5762026-03-19T14:39:59.340ZLINK OVER SPEED MACADDRESS MACADDRTYPE VID
5772026-03-19T14:39:59.340Zsc0_1 igb0 1000 a8:e1:de:1:70:1d fixed 0
5782026-03-19T14:39:59.340Z+ pfexec ipadm
5792026-03-19T14:39:59.340ZADDROBJ TYPE STATE ADDR
5802026-03-19T14:39:59.341Zlo0/v4 static ok 127.0.0.1/8
5812026-03-19T14:39:59.341Zigb0/dhcp dhcp ok 10.151.2.100/24
5822026-03-19T14:39:59.341Zlo0/v6 static ok ::1/128
5832026-03-19T14:39:59.341Z+ pfexec netstat -rncva
5842026-03-19T14:39:59.341Z
5852026-03-19T14:39:59.341ZIRE Table: IPv4
5862026-03-19T14:39:59.341Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
5872026-03-19T14:39:59.341Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
5882026-03-19T14:39:59.341Z0.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
5892026-03-19T14:39:59.341Zdefault 0.0.0.0 10.151.2.1 igb0 1500 9 UG 1654478 0
5902026-03-19T14:39:59.341Z10.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
5912026-03-19T14:39:59.341Z10.151.2.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
5922026-03-19T14:39:59.341Z10.151.2.0/24 255.255.255.0 10.151.2.100 igb0 1500 3 U 0 0
5932026-03-19T14:39:59.341Z10.151.2.1/32 255.255.255.255 10.151.2.100 igb0 1500 3 UC 0 0
5942026-03-19T14:39:59.341Z10.151.2.100 255.255.255.255 -- igb0 1500 2 UHL 2 3308240
5952026-03-19T14:39:59.341Z10.151.2.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
5962026-03-19T14:39:59.341Z10.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
5972026-03-19T14:39:59.341Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 14 14
5982026-03-19T14:39:59.341Z255.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 5
5992026-03-19T14:39:59.341Z
6002026-03-19T14:39:59.341ZIRE Table: IPv6
6012026-03-19T14:39:59.341Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
6022026-03-19T14:39:59.341Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
6032026-03-19T14:39:59.341Z::1 ::1 lo0 8252 2 UH 0 0
6042026-03-19T14:39:59.341Z+ pfexec netstat -anu
6052026-03-19T14:39:59.341Z
6062026-03-19T14:39:59.341ZUDP: IPv4
6072026-03-19T14:39:59.341Z Local Address Remote Address User Pid Command State
6082026-03-19T14:39:59.341Z-------------------- -------------------- -------- ------ -------------- ----------
6092026-03-19T14:39:59.341Z *.* root 71 in.mpathd Unbound
6102026-03-19T14:39:59.341Z *.* root 71 in.mpathd Unbound
6112026-03-19T14:39:59.341Z *.68 root 293 dhcpagent Idle
6122026-03-19T14:39:59.341Z *.546 root 293 dhcpagent Idle
6132026-03-19T14:39:59.341Z10.151.2.100.68 root 293 dhcpagent Idle
6142026-03-19T14:39:59.341Z *.111 daemon 710 rpcbind Idle
6152026-03-19T14:39:59.341Z *.* daemon 710 rpcbind Unbound
6162026-03-19T14:39:59.341Z *.51487 daemon 710 rpcbind Idle
6172026-03-19T14:39:59.341Z *.111 daemon 710 rpcbind Idle
6182026-03-19T14:39:59.341Z *.* daemon 710 rpcbind Unbound
6192026-03-19T14:39:59.342Z *.41826 daemon 710 rpcbind Idle
6202026-03-19T14:39:59.342Z *.* root 721 in.ndpd Unbound
6212026-03-19T14:39:59.342Z *.* root 868 rcm_daemon Unbound
6222026-03-19T14:39:59.342Z *.* root 868 rcm_daemon Unbound
6232026-03-19T14:39:59.342Z
6242026-03-19T14:39:59.342ZUDP: IPv6
6252026-03-19T14:39:59.342Z Local Address Remote Address User Pid Command State If
6262026-03-19T14:39:59.342Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
6272026-03-19T14:39:59.342Z *.* root 71 in.mpathd Unbound
6282026-03-19T14:39:59.342Z *.546 root 293 dhcpagent Idle
6292026-03-19T14:39:59.342Z *.111 daemon 710 rpcbind Idle
6302026-03-19T14:39:59.342Z *.* daemon 710 rpcbind Unbound
6312026-03-19T14:39:59.342Z *.51487 daemon 710 rpcbind Idle
6322026-03-19T14:39:59.342Z *.* root 721 in.ndpd Unbound
6332026-03-19T14:39:59.342Z *.* root 868 rcm_daemon Unbound
6342026-03-19T14:39:59.342Z
6352026-03-19T14:39:59.342ZTCP: IPv4
6362026-03-19T14:39:59.342Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State
6372026-03-19T14:39:59.342Z-------------------- -------------------- -------- ------ -------------- ------ ------ ------ ------ -----------
6382026-03-19T14:39:59.342Z127.0.0.1.5999 *.* root 71 in.mpathd 0 0 128000 0 LISTEN
6392026-03-19T14:39:59.342Z127.0.0.1.4999 *.* root 293 dhcpagent 0 0 128000 0 LISTEN
6402026-03-19T14:39:59.342Z *.111 *.* daemon 710 rpcbind 0 0 128000 0 LISTEN
6412026-03-19T14:39:59.342Z *.* *.* daemon 710 rpcbind 0 0 128000 0 IDLE
6422026-03-19T14:39:59.342Z *.111 *.* daemon 710 rpcbind 0 0 128000 0 LISTEN
6432026-03-19T14:39:59.342Z *.* *.* daemon 710 rpcbind 0 0 128000 0 IDLE
6442026-03-19T14:39:59.342Z10.151.2.100.57664 54.187.123.254.443 root 802 agent 1049792 0 128872 0 ESTABLISHED
6452026-03-19T14:39:59.342Z10.151.2.100.36976 54.187.123.254.443 root 802 agent 1049792 1366 128872 0 ESTABLISHED
6462026-03-19T14:39:59.342Z10.151.2.100.36217 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6472026-03-19T14:39:59.342Z10.151.2.100.46420 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6482026-03-19T14:39:59.342Z10.151.2.100.47877 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6492026-03-19T14:39:59.342Z10.151.2.100.59395 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
6502026-03-19T14:39:59.342Z10.151.2.100.46046 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
6512026-03-19T14:39:59.342Z
6522026-03-19T14:39:59.342ZTCP: IPv6
6532026-03-19T14:39:59.342Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State If
6542026-03-19T14:39:59.342Z--------------------------------- --------------------------------- -------- ------ -------------- ------ ------ ------ ------ ----------- -----
6552026-03-19T14:39:59.342Z::1.5999 *.* root 71 in.mpathd 0 0 128000 0 LISTEN
6562026-03-19T14:39:59.342Z *.111 *.* daemon 710 rpcbind 0 0 128000 0 LISTEN
6572026-03-19T14:39:59.343Z *.* *.* daemon 710 rpcbind 0 0 128000 0 IDLE
6582026-03-19T14:39:59.343Z
6592026-03-19T14:39:59.343ZActive UNIX domain sockets
6602026-03-19T14:39:59.343ZAddress Type User Pid Command Local Address Remote Address
6612026-03-19T14:39:59.343Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
6622026-03-19T14:39:59.343Zfffffe859e9453c8 stream-ord root 888 zoneadmd /var/run/zones/sidecar_softnpu.console_
6632026-03-19T14:39:59.343Zfffffe8586ed3758 stream-ord root 802 agent /var/run/buildomat.sock
6642026-03-19T14:39:59.343Zfffffe8586ed3af8 stream-ord root 802 agent
6652026-03-19T14:39:59.343Zfffffe8586ed3af8 stream-ord root 802 agent
6662026-03-19T14:39:59.343Zfffffe8586e99010 stream-ord root 802 agent
6672026-03-19T14:39:59.343Zfffffe85845173a8 stream-ord root 757 inetd /var/run/.inetd.uds
6682026-03-19T14:39:59.343Zfffffe8584517748 dgram root 721 in.ndpd /var/run/in.ndpd_mib
6692026-03-19T14:39:59.343Zfffffe8584517ae8 stream-ord root 721 in.ndpd /var/run/in.ndpd_ipadm
6702026-03-19T14:39:59.343Z+ pfexec arp -an
6712026-03-19T14:39:59.368ZNet to Media Table: IPv4
6722026-03-19T14:39:59.368ZDevice IP Address Mask Flags Phys Addr
6732026-03-19T14:39:59.368Z------ -------------------- --------------- -------- ---------------
6742026-03-19T14:39:59.368Zigb0 10.151.2.1 255.255.255.255 aa:00:04:00:ca:fe
6752026-03-19T14:39:59.368Zigb0 10.151.2.100 255.255.255.255 SPLA 18:c0:4d:0d:a0:2a
6762026-03-19T14:39:59.368Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
6772026-03-19T14:39:59.368Zlocal v6:
6782026-03-19T14:39:59.368Zlocal v4:
6792026-03-19T14:39:59.368Zrouter v6_idx:
6802026-03-19T14:39:59.368Zrouter v6_routes:
6812026-03-19T14:39:59.368Zrouter v4_idx:
6822026-03-19T14:39:59.368Zrouter v4_routes:
6832026-03-19T14:39:59.368Zresolver v4:
6842026-03-19T14:39:59.368Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
6852026-03-19T14:39:59.368Zresolver v6:
6862026-03-19T14:39:59.368Znat v4:
6872026-03-19T14:39:59.368Znat_v6:
6882026-03-19T14:39:59.368Zattached subnet v4:
6892026-03-19T14:39:59.368Zattached subnet v6:
6902026-03-19T14:39:59.368Zport_mac:
6912026-03-19T14:39:59.368Zproxy arp:
6922026-03-19T14:39:59.368Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
6932026-03-19T14:39:59.368Z+ pfexec /opt/oxide/opte/bin/opteadm list-ports
6942026-03-19T14:39:59.368ZLINK MAC ADDRESS IPv4 ADDRESS EPHEMERAL IPv4 FLOATING IPv4 IPv6 ADDRESS EXTERNAL IPv6 FLOATING IPv6 STATE
6952026-03-19T14:39:59.368Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2b
6962026-03-19T14:39:59.368ZVirtual to Boundary Mappings
6972026-03-19T14:39:59.368Z======================================================================
6982026-03-19T14:39:59.368Z
6992026-03-19T14:39:59.369ZIPv4 mappings
7002026-03-19T14:39:59.369Z----------------------------------------------------------------------
7012026-03-19T14:39:59.369ZTUNNELED PREFIX BOUNDARY IP VNI
7022026-03-19T14:39:59.369Z
7032026-03-19T14:39:59.369ZIPv6 mappings
7042026-03-19T14:39:59.369Z----------------------------------------------------------------------
7052026-03-19T14:39:59.369ZTUNNELED PREFIX BOUNDARY IP VNI
7062026-03-19T14:39:59.369Z
7072026-03-19T14:39:59.369Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2p
7082026-03-19T14:39:59.369ZVirtual to Physical Mappings
7092026-03-19T14:39:59.369Z======================================================================
7102026-03-19T14:39:59.393Z+ z_swadm link ls
7112026-03-19T14:39:59.393Z+ echo '== swadm link' ls
7122026-03-19T14:39:59.393Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm link ls
7132026-03-19T14:39:59.393Z== swadm link ls
7142026-03-19T14:39:59.393Zzlogin: zone 'oxz_switch' unknown
7152026-03-19T14:39:59.393Z+ z_swadm addr list
7162026-03-19T14:39:59.393Z+ echo '== swadm addr' list
7172026-03-19T14:39:59.393Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm addr list
7182026-03-19T14:39:59.393Z== swadm addr list
7192026-03-19T14:39:59.393Zzlogin: zone 'oxz_switch' unknown
7202026-03-19T14:39:59.393Z+ z_swadm route list
7212026-03-19T14:39:59.393Z+ echo '== swadm route' list
7222026-03-19T14:39:59.394Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm route list
7232026-03-19T14:39:59.394Z== swadm route list
7242026-03-19T14:39:59.394Zzlogin: zone 'oxz_switch' unknown
7252026-03-19T14:39:59.394Z+ z_swadm arp list
7262026-03-19T14:39:59.394Z+ echo '== swadm arp' list
7272026-03-19T14:39:59.394Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm arp list
7282026-03-19T14:39:59.394Z== swadm arp list
7292026-03-19T14:39:59.394Zzlogin: zone 'oxz_switch' unknown
7302026-03-19T14:39:59.394Z+ z_swadm nat list
7312026-03-19T14:39:59.394Z+ echo '== swadm nat' list
7322026-03-19T14:39:59.394Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm nat list
7332026-03-19T14:39:59.394Z== swadm nat list
7342026-03-19T14:39:59.394Zzlogin: zone 'oxz_switch' unknown
7352026-03-19T14:39:59.394Z++ pfexec /opt/oxide/opte/bin/opteadm list-ports
7362026-03-19T14:39:59.394Z++ tail +2
7372026-03-19T14:39:59.394Z++ awk '{ print $1; }'
7382026-03-19T14:39:59.418Z+ PORTS=
7392026-03-19T14:39:59.419Z+ mkdir -p /tmp/kstat
7402026-03-19T14:39:59.419Z+ pfexec kstat -p xde:
7412026-03-19T14:39:59.419Z+ pfexec zfs list
7422026-03-19T14:39:59.419ZNAME USED AVAIL REFER MOUNTPOINT
7432026-03-19T14:39:59.419Zrpool 2.24G 5.03G 24K /rpool
7442026-03-19T14:39:59.419Zrpool/home 25.5K 5.03G 25.5K /home
7452026-03-19T14:39:59.419Zrpool/input 2.16G 5.03G 2.16G /input
7462026-03-19T14:39:59.419Zrpool/sidecar 75.8M 5.03G 24K /sidecar
7472026-03-19T14:39:59.419Zrpool/sidecar/sidecar_softnpu 75.8M 5.03G 75.8M /sidecar/sidecar_softnpu
7482026-03-19T14:39:59.419Zrpool/work 24K 5.03G 24K /work
7492026-03-19T14:39:59.419Zrpool/zone 24K 5.03G 24K /zone
7502026-03-19T14:39:59.419Zscratch 1.37M 1.80T 25K /scratch
7512026-03-19T14:39:59.419Z+ pfexec zpool list
7522026-03-19T14:39:59.419ZNAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
7532026-03-19T14:39:59.419Zrpool 7.50G 2.24G 5.26G - - 0% 29% 1.00x ONLINE -
7542026-03-19T14:39:59.419Zscratch 1.86T 1.37M 1.86T - - 0% 0% 1.00x ONLINE -
7552026-03-19T14:39:59.419Z+ pfexec fmdump -eVp
7562026-03-19T14:39:59.443ZTIME CLASS
7572026-03-19T14:39:59.443Z+ pfexec ptree -z global
7582026-03-19T14:39:59.443Z8 /lib/svc/bin/svc.startd
7592026-03-19T14:39:59.443Z 639 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
7602026-03-19T14:39:59.443Z 651 /usr/lib/saf/sac -t 300
7612026-03-19T14:39:59.443Z 654 /usr/lib/saf/ttymon
7622026-03-19T14:39:59.443Z 801 /usr/bin/ctrun -l child -o noorphan,regent /opt/buildomat/lib/agent ru
7632026-03-19T14:39:59.443Z 802 /opt/buildomat/lib/agent run
7642026-03-19T14:39:59.443Z 816 /bin/bash /tmp/PAomy9ty8jPeK47A.sh
7652026-03-19T14:39:59.444Z 1660 ptree -z global
7662026-03-19T14:39:59.444Z10 /lib/svc/bin/svc.configd
7672026-03-19T14:39:59.444Z38 /lib/inet/ipmgmtd
7682026-03-19T14:39:59.444Z40 /sbin/dlmgmtd
7692026-03-19T14:39:59.444Z43 /lib/inet/netcfgd
7702026-03-19T14:39:59.444Z71 /lib/inet/in.mpathd
7712026-03-19T14:39:59.444Z91 /usr/lib/pfexecd
7722026-03-19T14:39:59.444Z142 /usr/lib/zones/zonestatd
7732026-03-19T14:39:59.444Z171 /usr/lib/sysevent/syseventd
7742026-03-19T14:39:59.444Z185 devfsadmd
7752026-03-19T14:39:59.444Z293 /sbin/dhcpagent
7762026-03-19T14:39:59.444Z563 /usr/lib/picl/picld
7772026-03-19T14:39:59.444Z597 /usr/sbin/nscd
7782026-03-19T14:39:59.444Z638 /usr/lib/utmpd
7792026-03-19T14:39:59.444Z664 /usr/lib/hotplugd
7802026-03-19T14:39:59.444Z685 /usr/sbin/syslogd
7812026-03-19T14:39:59.444Z710 /usr/sbin/rpcbind
7822026-03-19T14:39:59.444Z721 /usr/lib/inet/in.ndpd
7832026-03-19T14:39:59.444Z722 /usr/lib/fm/fmd/fmd
7842026-03-19T14:39:59.444Z757 /usr/lib/inet/inetd start
7852026-03-19T14:39:59.444Z779 /usr/perl5/bin/perl /usr/lib/intrd
7862026-03-19T14:39:59.444Z868 /usr/lib/rcm/rcm_daemon
7872026-03-19T14:39:59.444Z888 zoneadmd -z sidecar_softnpu
7882026-03-19T14:39:59.444Z+ pfexec svcs -xv
7892026-03-19T14:39:59.493Zsvc:/oxide/sled-agent:default (Oxide Sled Agent)
7902026-03-19T14:39:59.493Z State: maintenance since March 19, 2026 at 02:39:27 PM UTC
7912026-03-19T14:39:59.493ZReason: Restarting too quickly.
7922026-03-19T14:39:59.493Z See: http://illumos.org/msg/SMF-8000-L5
7932026-03-19T14:39:59.493Z See: /var/svc/log/oxide-sled-agent:default.log
7942026-03-19T14:39:59.493ZImpact: This service is not running.
7952026-03-19T14:39:59.519Z++ zoneadm list -n
7962026-03-19T14:39:59.519Z+ for z in $(zoneadm list -n)
7972026-03-19T14:39:59.519Z+ banner sidecar_softnpu
7982026-03-19T14:39:59.519Z
7992026-03-19T14:39:59.519Z #### # ##### ###### #### ## ##### #### ####
8002026-03-19T14:39:59.519Z # # # # # # # # # # # # # #
8012026-03-19T14:39:59.519Z #### # # # ##### # # # # # #### # #
8022026-03-19T14:39:59.519Z # # # # # # ###### ##### # # #
8032026-03-19T14:39:59.519Z # # # # # # # # # # # # # # # #
8042026-03-19T14:39:59.519Z #### # ##### ###### #### # # # # ####### #### ####
8052026-03-19T14:39:59.520Z
8062026-03-19T14:39:59.520Z+ pfexec svcs -xv -z sidecar_softnpu
8072026-03-19T14:39:59.588Z+ pfexec ptree -z sidecar_softnpu
8082026-03-19T14:39:59.614Z891 zsched
8092026-03-19T14:39:59.614Z 908 /sbin/init
8102026-03-19T14:39:59.614Z913 /lib/svc/bin/svc.startd
8112026-03-19T14:39:59.614Z 1442 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
8122026-03-19T14:39:59.614Z915 /lib/svc/bin/svc.configd
8132026-03-19T14:39:59.614Z958 /lib/inet/ipmgmtd
8142026-03-19T14:39:59.614Z963 /lib/inet/netcfgd
8152026-03-19T14:39:59.614Z1166 /usr/lib/fm/fmd/fmd
8162026-03-19T14:39:59.614Z1182 /usr/lib/pfexecd
8172026-03-19T14:39:59.614Z1289 /softnpu/softnpu --uds-path /softnpu /softnpu/softnpu.toml
8182026-03-19T14:39:59.614Z1389 /usr/sbin/nscd
8192026-03-19T14:39:59.614Z1440 /usr/lib/utmpd
8202026-03-19T14:39:59.615Z1465 /usr/sbin/syslogd
8212026-03-19T14:39:59.615Z1513 /usr/lib/inet/in.ndpd
8222026-03-19T14:39:59.615Z+ pfexec zlogin sidecar_softnpu ipadm
8232026-03-19T14:39:59.615ZADDROBJ TYPE STATE ADDR
8242026-03-19T14:39:59.615Zlo0/v4 static ok 127.0.0.1/8
8252026-03-19T14:39:59.615Zlo0/v6 static ok ::1/128
8262026-03-19T14:39:59.615Z+ pfexec zlogin sidecar_softnpu netstat -rncva
8272026-03-19T14:39:59.615Z
8282026-03-19T14:39:59.615ZIRE Table: IPv4
8292026-03-19T14:39:59.615Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
8302026-03-19T14:39:59.615Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
8312026-03-19T14:39:59.615Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 0 0
8322026-03-19T14:39:59.615Z
8332026-03-19T14:39:59.615ZIRE Table: IPv6
8342026-03-19T14:39:59.615Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
8352026-03-19T14:39:59.615Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
8362026-03-19T14:39:59.615Z::1 ::1 lo0 8252 2 UH 0 0
8372026-03-19T14:39:59.615Z+ pfexec zlogin sidecar_softnpu netstat -anu
8382026-03-19T14:39:59.641Z
8392026-03-19T14:39:59.641ZUDP: IPv4
8402026-03-19T14:39:59.641Z Local Address Remote Address User Pid Command State
8412026-03-19T14:39:59.641Z-------------------- -------------------- -------- ------ -------------- ----------
8422026-03-19T14:39:59.641Z *.* root 1513 in.ndpd Unbound
8432026-03-19T14:39:59.641Z
8442026-03-19T14:39:59.641ZUDP: IPv6
8452026-03-19T14:39:59.641Z Local Address Remote Address User Pid Command State If
8462026-03-19T14:39:59.641Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
8472026-03-19T14:39:59.641Z *.* root 1513 in.ndpd Unbound
8482026-03-19T14:39:59.641Z
8492026-03-19T14:39:59.641ZActive UNIX domain sockets
8502026-03-19T14:39:59.642ZAddress Type User Pid Command Local Address Remote Address
8512026-03-19T14:39:59.642Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
8522026-03-19T14:39:59.642Zfffffe85a4c96b20 dgram root 1513 in.ndpd /var/run/in.ndpd_mib
8532026-03-19T14:39:59.642Zfffffe85a6505038 stream-ord root 1513 in.ndpd /var/run/in.ndpd_ipadm
8542026-03-19T14:39:59.642Zfffffe859afb3020 stream-ord root 1289 softnpu
8552026-03-19T14:39:59.642Zfffffe859afb3020 stream-ord root 1289 softnpu
8562026-03-19T14:39:59.642Zfffffe859afb33c0 stream-ord root 1289 softnpu
8572026-03-19T14:39:59.642Zfffffe859afb3760 dgram root 1289 softnpu /softnpu/server
8582026-03-19T14:39:59.642Z+ pfexec zlogin sidecar_softnpu arp -an
8592026-03-19T14:39:59.666Z++ zoneadm list -n
8602026-03-19T14:39:59.666Z++ grep oxz_ntp
8612026-03-19T14:39:59.666Z+ pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log
8622026-03-19T14:39:59.691ZConfig {
8632026-03-19T14:39:59.691Z p4_program: "/softnpu/asic_program.so",
8642026-03-19T14:39:59.691Z ports: [
8652026-03-19T14:39:59.691Z Port {
8662026-03-19T14:39:59.691Z sidecar: "sc0_0",
8672026-03-19T14:39:59.691Z scrimlet: "sidecar_shi0",
8682026-03-19T14:39:59.691Z mtu: 9000,
8692026-03-19T14:39:59.691Z },
8702026-03-19T14:39:59.691Z Port {
8712026-03-19T14:39:59.691Z sidecar: "sc0_1",
8722026-03-19T14:39:59.691Z scrimlet: "sidecar_shi1",
8732026-03-19T14:39:59.691Z mtu: 9000,
8742026-03-19T14:39:59.691Z },
8752026-03-19T14:39:59.691Z ],
8762026-03-19T14:39:59.691Z}
8772026-03-19T14:39:59.691Z+ exit 1
8782026-03-19T14:39:59.691Zprocess exited: duration 87191 ms, exit code 1
 
8792026-03-19T14:39:59.741Zfound 2 output files
8802026-03-19T14:39:59.741Zuploading: /var/svc/log/oxide-sled-agent:default.log (2990 bytes)
8812026-03-19T14:40:00.838Zuploaded: /var/svc/log/oxide-sled-agent:default.log
8822026-03-19T14:40:00.838Zuploading: /tmp/kstat/xde.kstat (890 bytes)
8832026-03-19T14:40:01.932Zuploaded: /tmp/kstat/xde.kstat