01KKXAP3DRP60AHJ64S9VM2Q44: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.39",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KKXAPA1N267W2K3KFWT5KMJF

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-03-17T07:20:47.244Zjob dependencies complete; ready to run (waiting for 0 s)
22026-03-17T07:21:57.391Zjob assigned to worker 01KKXAPFCRJAVTZQJH1JH7C12T [factory lab, sock/16330] (queued for 1 m 10 s)
32026-03-17T07:21:57.646Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-03-17T07:21:57.701Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52026-03-17T07:21:57.739Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62026-03-17T07:21:57.772Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72026-03-17T07:21:57.806Zhttp://172.20.0.5:9969/postboot/sock... ok
82026-03-17T07:21:57.839ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92026-03-17T07:21:57.872ZWARNING: cpu0: failed to update microcode from version 0xa001133 to 0xa0011de
102026-03-17T07:21:57.906ZWARNING: cpu0: using older fallback microcode; update the system firmware
112026-03-17T07:21:57.939Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d5
122026-03-17T07:21:57.973ZOxide Helios Version helios-2.0.23743 64-bit
132026-03-17T07:21:58.006ZConfiguring devices.
142026-03-17T07:21:58.040Z##################################################
152026-03-17T07:21:58.095Z#### ############# ##
162026-03-17T07:21:58.129Z### ### ############ ##
172026-03-17T07:21:58.162Z## ### # ## ### ## Oxide Computer Company ##
182026-03-17T07:21:58.195Z## ## ## ### # ### ##
192026-03-17T07:21:58.229Z## # ### #### #### This Station Under ##
202026-03-17T07:21:58.262Z### ### #### # ### Computer Control ##
212026-03-17T07:21:58.296Z#### #### ### ## ##
222026-03-17T07:21:58.329Z##################################################
232026-03-17T07:21:58.363ZHostname: sock
242026-03-17T07:21:58.396ZLoading smf(7) service descriptions:
252026-03-17T07:21:59.100Zdownloading input: /input/package/work/package.tar.gz
262026-03-17T07:29:09.660Zdownloaded input: /input/package/work/package.tar.gz
 
272026-03-17T07:29:09.685Zstarting task 0: "setup"
282026-03-17T07:29:09.685Z++ uname -s
292026-03-17T07:29:09.685Z+ kern=SunOS
302026-03-17T07:29:09.685Z+ build_user=build
312026-03-17T07:29:09.685Z+ build_uid=12345
322026-03-17T07:29:09.685Z+ work_dir=/work
332026-03-17T07:29:09.685Z+ input_dir=/input
342026-03-17T07:29:09.685Z+ [[ 0 == 12345 ]]
352026-03-17T07:29:09.686Z+ case "$kern" in
362026-03-17T07:29:09.686Z+ groupadd -g 12345 build
372026-03-17T07:29:09.686Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-03-17T07:29:11.679Z+ zfs create -o mountpoint=/work rpool/work
392026-03-17T07:29:11.725Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-03-17T07:29:11.750Z+ home_fs=zfs
412026-03-17T07:29:11.750Z+ [[ zfs == autofs ]]
422026-03-17T07:29:11.750Z+ mkdir -p /home/build
432026-03-17T07:29:11.750Z+ chown build:build /home/build /work
442026-03-17T07:29:13.734Z+ chmod 0700 /home/build /work
452026-03-17T07:29:13.759Zprocess exited: duration 4077 ms, exit code 0
 
462026-03-17T07:29:13.808Zstarting task 1: "authentication"
472026-03-17T07:29:13.852Zprocess exited: duration 44 ms, exit code 0
 
482026-03-17T07:29:13.900Zstarting task 2: "build"
492026-03-17T07:29:13.924Z+ trap _exit_trap EXIT
502026-03-17T07:29:13.924Z+ OPTE_COMMIT=
512026-03-17T07:29:13.924Z+ [[ x != \x ]]
522026-03-17T07:29:13.924Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
532026-03-17T07:29:14.017Zsecpolicy_fs_linkdir: 0x55 = 0x55
542026-03-17T07:29:14.017Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
552026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
562026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+3: 0 = 0xe5
572026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+4: 0 = 0xb8
582026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+5: 0 = 0x1
592026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
602026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
612026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
622026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
632026-03-17T07:29:14.018Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
642026-03-17T07:29:14.042Z+ [[ -d /opt/oxide ]]
652026-03-17T07:29:14.042Z+ pfexec mv /opt/oxide /opt/oxide-underneath
662026-03-17T07:29:14.042Z+ pfexec mkdir /opt/oxide
672026-03-17T07:29:14.042Z+ pfexec mount -F tmpfs -O swap /opt/oxide
682026-03-17T07:29:14.042Z+ [[ -d /opt/oxide-underneath ]]
692026-03-17T07:29:14.042Z+ cd /opt/oxide-underneath
702026-03-17T07:29:14.042Z+ pfexec tar ceEp@/f - .
712026-03-17T07:29:14.042Z+ cd /opt/oxide
722026-03-17T07:29:14.042Z+ pfexec tar xveEp@/f -
732026-03-17T07:29:14.085Zx ., 0 bytes, 0 tape blocks
742026-03-17T07:29:14.085Zx ./opte, 0 bytes, 0 tape blocks
752026-03-17T07:29:14.085Zx ./opte/bin, 0 bytes, 0 tape blocks
762026-03-17T07:29:14.085Zx ./opte/bin/opteadm, 22792392 bytes, 44517 tape blocks
772026-03-17T07:29:14.110Z+ pfexec rm -rf /opt/oxide-underneath
782026-03-17T07:29:14.110Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
792026-03-17T07:29:14.110Z+ pfexec mkdir /opt/oxide/work
802026-03-17T07:29:14.135Z+ pfexec chown build:build /opt/oxide/work
812026-03-17T07:29:14.135Z+ cd /opt/oxide/work
822026-03-17T07:29:14.135Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
832026-03-17T07:29:35.672ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
842026-03-17T07:29:35.673Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
852026-03-17T07:29:35.673Zx out/target/test, 107 bytes, 1 tape blocks
862026-03-17T07:29:35.673Zx out/npuzone/npuzone, 28059200 bytes, 54804 tape blocks
872026-03-17T07:29:35.673Zx package-manifest.toml, 34895 bytes, 69 tape blocks
882026-03-17T07:29:35.673Zx smf/sled-agent/non-gimlet/config.toml, 5328 bytes, 11 tape blocks
892026-03-17T07:29:35.673Zx target/release/omicron-package, 41908560 bytes, 81853 tape blocks
902026-03-17T07:29:35.673Zx target/release/xtask, 11699728 bytes, 22852 tape blocks
912026-03-17T07:29:35.673Zx target/debug/bootstrap, 343921080 bytes, 671721 tape blocks
922026-03-17T07:29:35.673Zx tests/bootstrap-ce927ede589904ba, 32061232 bytes, 62620 tape blocks
932026-03-17T07:29:35.673Zx tests/commtest-ed680c524300f199, 33172184 bytes, 64790 tape blocks
942026-03-17T07:29:35.673Zx tests/dhcp_server-530ecc14a0c4d124, 32061248 bytes, 62620 tape blocks
952026-03-17T07:29:35.673Zx tests/end_to_end_tests-3148bac0dcf3127f, 386821536 bytes, 755511 tape blocks
962026-03-17T07:29:35.673Zx out/clickhouse.tar.gz, 325126989 bytes, 635014 tape blocks
972026-03-17T07:29:35.673Zx out/clickhouse_keeper.tar.gz, 308717586 bytes, 602965 tape blocks
982026-03-17T07:29:35.673Zx out/clickhouse_server.tar.gz, 325125809 bytes, 635012 tape blocks
992026-03-17T07:29:35.673Zx out/cockroachdb.tar.gz, 169974835 bytes, 331983 tape blocks
1002026-03-17T07:29:35.673Zx out/crucible-pantry-zone.tar.gz, 43734698 bytes, 85420 tape blocks
1012026-03-17T07:29:35.674Zx out/crucible-utils.tar, 18169344 bytes, 35487 tape blocks
1022026-03-17T07:29:35.674Zx out/crucible-zone.tar.gz, 58212119 bytes, 113696 tape blocks
1032026-03-17T07:29:35.674Zx out/external-dns.tar.gz, 51742204 bytes, 101059 tape blocks
1042026-03-17T07:29:35.674Zx out/internal-dns.tar.gz, 51741800 bytes, 101059 tape blocks
1052026-03-17T07:29:35.674Zx out/mg-ddm-gz.tar, 80258048 bytes, 156754 tape blocks
1062026-03-17T07:29:35.674Zx out/nexus.tar.gz, 221900226 bytes, 433399 tape blocks
1072026-03-17T07:29:35.674Zx out/ntp.tar.gz, 31911439 bytes, 62328 tape blocks
1082026-03-17T07:29:35.674Zx out/omicron-gateway-softnpu.tar.gz, 27988426 bytes, 54665 tape blocks
1092026-03-17T07:29:35.674Zx out/omicron-sled-agent.tar, 205324800 bytes, 401025 tape blocks
1102026-03-17T07:29:35.674Zx out/overlay.tar.gz, 3030 bytes, 6 tape blocks
1112026-03-17T07:29:35.674Zx out/oximeter.tar.gz, 67399776 bytes, 131641 tape blocks
1122026-03-17T07:29:35.674Zx out/oxlog.tar, 3871744 bytes, 7562 tape blocks
1132026-03-17T07:29:35.674Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1142026-03-17T07:29:35.674Zx out/propolis-server.tar.gz, 76424187 bytes, 149266 tape blocks
1152026-03-17T07:29:35.674Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1162026-03-17T07:29:35.674Zx out/switch-softnpu.tar.gz, 388406909 bytes, 758608 tape blocks
1172026-03-17T07:29:35.703Z
1182026-03-17T07:29:35.703Zreal 21.530251859
1192026-03-17T07:29:35.703Zuser 19.279707725
1202026-03-17T07:29:35.703Zsys 3.798248965
1212026-03-17T07:29:35.703Ztrap 0.000911267
1222026-03-17T07:29:35.703Ztflt 0.000000000
1232026-03-17T07:29:35.703Zdflt 0.000000000
1242026-03-17T07:29:35.703Zkflt 0.000000000
1252026-03-17T07:29:35.703Zlock 0.000000000
1262026-03-17T07:29:35.704Zslp 19.425857963
1272026-03-17T07:29:35.704Zlat 0.553477172
1282026-03-17T07:29:35.704Zstop 0.000138090
1292026-03-17T07:29:35.704Z+ source .github/buildomat/ci-env.sh
1302026-03-17T07:29:35.704Z++ export CARGO_TERM_COLOR=always
1312026-03-17T07:29:35.704Z++ CARGO_TERM_COLOR=always
1322026-03-17T07:29:35.704Z+ bmat address ls
1332026-03-17T07:29:35.704ZNAME CIDR FIRST LAST COUNT
1342026-03-17T07:29:35.704Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1352026-03-17T07:29:35.704Z++ bmat address ls -f extra -Ho first,last,count
1362026-03-17T07:29:35.704Z+ set -- 10.151.2.164 10.151.2.227 64
1372026-03-17T07:29:35.704Z+ EXTRA_IP_START=10.151.2.164
1382026-03-17T07:29:35.704Z+ EXTRA_IP_END=10.151.2.227
1392026-03-17T07:29:35.704Z+ EXTRA_IP_COUNT=64
1402026-03-17T07:29:35.704Z+ (( EXTRA_IP_COUNT >= 32 ))
1412026-03-17T07:29:35.704Z+ EXTRA_IP_BASE=10.151.2
1422026-03-17T07:29:35.704Z+ EXTRA_IP_FOCTET=164
1432026-03-17T07:29:35.704Z+ EXTRA_IP_LOCTET=227
1442026-03-17T07:29:35.704Z+ SERVICE_IP_POOL_START=10.151.2.164
1452026-03-17T07:29:35.704Z+ SERVICE_IP_POOL_END=10.151.2.173
1462026-03-17T07:29:35.704Z+ DNS_IP1=10.151.2.164
1472026-03-17T07:29:35.705Z+ DNS_IP2=10.151.2.165
1482026-03-17T07:29:35.705Z+ UPLINK_IP=10.151.2.174
1492026-03-17T07:29:35.705Z+ IPPOOL_START=10.151.2.175
1502026-03-17T07:29:35.705Z+ IPPOOL_END=10.151.2.227
1512026-03-17T07:29:35.705Z++ ipadm show-addr -po type,addr
1522026-03-17T07:29:35.705Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1532026-03-17T07:29:35.705Z+ GATEWAY_IP=10.151.2.100
1542026-03-17T07:29:35.705Z+ [[ -n 10.151.2.100 ]]
1552026-03-17T07:29:35.705Z+ ping -s 10.151.2.100 56 1
1562026-03-17T07:29:35.705ZPING 10.151.2.100: 56 data bytes
1572026-03-17T07:29:35.705Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.011 ms
1582026-03-17T07:29:35.705Z
1592026-03-17T07:29:35.705Z----10.151.2.100 PING Statistics----
1602026-03-17T07:29:35.706Z1 packets transmitted, 1 packets received, 0% packet loss
1612026-03-17T07:29:35.706Zround-trip (ms) min/avg/max/stddev = 0.011/0.011/0.011/-nan
1622026-03-17T07:29:35.706Z++ arp -an
1632026-03-17T07:29:35.706Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1642026-03-17T07:29:35.706Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1652026-03-17T07:29:35.706Z+ routeadm -e ipv4-forwarding -u
1662026-03-17T07:29:35.798Z+ PXA_START=10.151.2.164
1672026-03-17T07:29:35.798Z+ PXA_END=10.151.2.227
1682026-03-17T07:29:35.798Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
1692026-03-17T07:29:35.935Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1702026-03-17T07:29:35.962ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1712026-03-17T07:29:36.009Zcreating virtual hardware
1722026-03-17T07:29:36.035Zcreating /scratch/m2_0.vdev
1732026-03-17T07:29:36.035Zcreating /scratch/m2_1.vdev
1742026-03-17T07:29:36.035Zcreating /scratch/u2_0.vdev
1752026-03-17T07:29:36.035Zcreating /scratch/u2_1.vdev
1762026-03-17T07:29:36.035Zcreating /scratch/u2_2.vdev
1772026-03-17T07:29:36.035Zcreating /scratch/u2_3.vdev
1782026-03-17T07:29:36.035Zcreating /scratch/u2_4.vdev
1792026-03-17T07:29:36.035Zcreating /scratch/u2_5.vdev
1802026-03-17T07:29:36.035Zcreating /scratch/u2_6.vdev
1812026-03-17T07:29:36.035Zcreating /scratch/u2_7.vdev
1822026-03-17T07:29:36.035Zcreating /scratch/u2_8.vdev
1832026-03-17T07:29:36.060ZSimnet net0/sc0_0 exists
1842026-03-17T07:29:36.060ZSimnet net1/sc1_0 exists
1852026-03-17T07:29:36.086ZVnic sc0_1 exists
1862026-03-17T07:29:49.950ZUsing 10.151.2.100 as gateway ip
1872026-03-17T07:29:49.950Zusing 18:C0:4D:0D:A0:2A as gateway mac
1882026-03-17T07:29:49.950Zconfiguring SoftNPU ARP entry
1892026-03-17T07:29:49.950Zconfiguring SoftNPU proxy ARP
1902026-03-17T07:29:49.950ZSoftNPU state:
1912026-03-17T07:29:49.950Z local v6:
1922026-03-17T07:29:49.950Z local v4:
1932026-03-17T07:29:49.950Z router v6_idx:
1942026-03-17T07:29:49.950Z router v6_routes:
1952026-03-17T07:29:49.950Z router v4_idx:
1962026-03-17T07:29:49.950Z router v4_routes:
1972026-03-17T07:29:49.950Z resolver v4:
1982026-03-17T07:29:49.950Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
1992026-03-17T07:29:49.950Z resolver v6:
2002026-03-17T07:29:49.950Z nat v4:
2012026-03-17T07:29:49.950Z nat_v6:
2022026-03-17T07:29:49.950Z attached subnet v4:
2032026-03-17T07:29:49.950Z attached subnet v6:
2042026-03-17T07:29:49.951Z port_mac:
2052026-03-17T07:29:49.951Z proxy arp:
2062026-03-17T07:29:49.951Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2072026-03-17T07:29:49.951Zcreated virtual hardware
2082026-03-17T07:29:49.951Z
2092026-03-17T07:29:49.951Zreal 13.370143366
2102026-03-17T07:29:49.951Zuser 0.657036623
2112026-03-17T07:29:50.001Zsys 3.332030656
2122026-03-17T07:29:50.001Ztrap 0.000113064
2132026-03-17T07:29:50.001Ztflt 0.000431750
2142026-03-17T07:29:50.001Zdflt 0.000677736
2152026-03-17T07:29:50.001Zkflt 0.000010580
2162026-03-17T07:29:50.001Zlock 10:21.546453329
2172026-03-17T07:29:50.001Zslp 40.355391127
2182026-03-17T07:29:50.002Zlat 0.121354134
2192026-03-17T07:29:50.002Zstop 0.245712713
2202026-03-17T07:29:50.002Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2212026-03-17T07:29:50.002Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2222026-03-17T07:29:50.002Z+ diff -u pkg/config.toml~ pkg/config.toml
2232026-03-17T07:29:50.002Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2242026-03-17T07:29:50.002Z+++ pkg/config.toml Tue Mar 17 07:29:49 2026
2252026-03-17T07:29:50.002Z@@ -29,18 +29,18 @@
2262026-03-17T07:29:50.002Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2272026-03-17T07:29:50.002Z # followed by a string that is embedded into their fake serial values.
2282026-03-17T07:29:50.002Z vdevs = [
2292026-03-17T07:29:50.002Z- "m2_0.vdev",
2302026-03-17T07:29:50.002Z- "m2_1.vdev",
2312026-03-17T07:29:50.002Z+ "/scratch/m2_0.vdev",
2322026-03-17T07:29:50.002Z+ "/scratch/m2_1.vdev",
2332026-03-17T07:29:50.002Z
2342026-03-17T07:29:50.002Z- "u2_0.vdev",
2352026-03-17T07:29:50.002Z- "u2_1.vdev",
2362026-03-17T07:29:50.002Z- "u2_2.vdev",
2372026-03-17T07:29:50.002Z- "u2_3.vdev",
2382026-03-17T07:29:50.002Z- "u2_4.vdev",
2392026-03-17T07:29:50.002Z- "u2_5.vdev",
2402026-03-17T07:29:50.002Z- "u2_6.vdev",
2412026-03-17T07:29:50.002Z- "u2_7.vdev",
2422026-03-17T07:29:50.002Z- "u2_8.vdev",
2432026-03-17T07:29:50.002Z+ "/scratch/u2_0.vdev",
2442026-03-17T07:29:50.002Z+ "/scratch/u2_1.vdev",
2452026-03-17T07:29:50.002Z+ "/scratch/u2_2.vdev",
2462026-03-17T07:29:50.002Z+ "/scratch/u2_3.vdev",
2472026-03-17T07:29:50.002Z+ "/scratch/u2_4.vdev",
2482026-03-17T07:29:50.002Z+ "/scratch/u2_5.vdev",
2492026-03-17T07:29:50.002Z+ "/scratch/u2_6.vdev",
2502026-03-17T07:29:50.003Z+ "/scratch/u2_7.vdev",
2512026-03-17T07:29:50.003Z+ "/scratch/u2_8.vdev",
2522026-03-17T07:29:50.003Z ]
2532026-03-17T07:29:50.003Z
2542026-03-17T07:29:50.003Z # The amount of memory held back for services which exist between zero and one
2552026-03-17T07:29:50.003Z+ true
2562026-03-17T07:29:50.003Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2572026-03-17T07:29:50.003Z+ EXPECTED_ZPOOL_COUNT=9
2582026-03-17T07:29:50.003Z+ echo 'expected number of zpools is 9'
2592026-03-17T07:29:50.003Zexpected number of zpools is 9
2602026-03-17T07:29:50.003Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2612026-03-17T07:29:50.003Z+ SILO_NAME=recovery
2622026-03-17T07:29:50.003Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2632026-03-17T07:29:50.003Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2642026-03-17T07:29:50.003Z+ sed -i~ '
2652026-03-17T07:29:50.003Z /^external_dns_ips/c\
2662026-03-17T07:29:50.003Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2672026-03-17T07:29:50.003Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2682026-03-17T07:29:50.003Z /^first/c\
2692026-03-17T07:29:50.003Zfirst = "10.151.2.164"
2702026-03-17T07:29:50.003Z /^last/c\
2712026-03-17T07:29:50.003Zlast = "10.151.2.173"
2722026-03-17T07:29:50.003Z }
2732026-03-17T07:29:50.003Z /^infra_ip_first/c\
2742026-03-17T07:29:50.003Zinfra_ip_first = "10.151.2.174"
2752026-03-17T07:29:50.003Z /^infra_ip_last/c\
2762026-03-17T07:29:50.003Zinfra_ip_last = "10.151.2.174"
2772026-03-17T07:29:50.003Z /^\[\[rack_network_config.ports/,/^$/ {
2782026-03-17T07:29:50.003Z /^routes/c\
2792026-03-17T07:29:50.003Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2802026-03-17T07:29:50.003Z /^addresses/c\
2812026-03-17T07:29:50.003Zaddresses = \[{address = "10.151.2.174/24"} \]
2822026-03-17T07:29:50.003Z }
2832026-03-17T07:29:50.003Z' pkg/config-rss.toml
2842026-03-17T07:29:50.003Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2852026-03-17T07:29:50.003Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2862026-03-17T07:29:50.003Z+++ pkg/config-rss.toml Tue Mar 17 07:29:49 2026
2872026-03-17T07:29:50.003Z@@ -33,7 +33,7 @@
2882026-03-17T07:29:50.003Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2892026-03-17T07:29:50.004Z+ true
2902026-03-17T07:29:50.004Z # must be contained in one of the "internal services" IP Pool ranges listed
2912026-03-17T07:29:50.004Z # below.
2922026-03-17T07:29:50.004Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2932026-03-17T07:29:50.004Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2942026-03-17T07:29:50.004Z
2952026-03-17T07:29:50.004Z # Initial TLS certificates for the external API
2962026-03-17T07:29:50.004Z #
2972026-03-17T07:29:50.004Z@@ -69,8 +69,8 @@
2982026-03-17T07:29:50.004Z #
2992026-03-17T07:29:50.004Z # For more on this and what to put here, see docs/how-to-run.adoc.
3002026-03-17T07:29:50.004Z [[internal_services_ip_pool_ranges]]
3012026-03-17T07:29:50.004Z-first = "192.168.1.20"
3022026-03-17T07:29:50.004Z-last = "192.168.1.29"
3032026-03-17T07:29:50.004Z+first = "10.151.2.164"
3042026-03-17T07:29:50.004Z+last = "10.151.2.173"
3052026-03-17T07:29:50.004Z
3062026-03-17T07:29:50.004Z # TODO - this configuration is subject to change going forward. Ultimately these
3072026-03-17T07:29:50.004Z # parameters should be provided to the control plane via wicket, but we need to
3082026-03-17T07:29:50.004Z@@ -91,8 +91,8 @@
3092026-03-17T07:29:50.004Z # A range of IP addresses used by Boundary Services on the external network. In
3102026-03-17T07:29:50.004Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3112026-03-17T07:29:50.029Z # With softnpu, only one address is used.
3122026-03-17T07:29:50.029Z-infra_ip_first = "192.168.1.30"
3132026-03-17T07:29:50.029Z-infra_ip_last = "192.168.1.30"
3142026-03-17T07:29:50.029Z+infra_ip_first = "10.151.2.174"
3152026-03-17T07:29:50.029Z+infra_ip_last = "10.151.2.174"
3162026-03-17T07:29:50.029Z
3172026-03-17T07:29:50.029Z # Configurations for BGP routers to run on the scrimlets.
3182026-03-17T07:29:50.029Z bgp = []
3192026-03-17T07:29:50.029Z@@ -100,9 +100,9 @@
3202026-03-17T07:29:50.029Z # You can configure multiple uplinks by repeating the following stanza
3212026-03-17T07:29:50.029Z [[rack_network_config.ports]]
3222026-03-17T07:29:50.029Z # Routes associated with this port.
3232026-03-17T07:29:50.029Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3242026-03-17T07:29:50.029Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3252026-03-17T07:29:50.029Z # Addresses associated with this port.
3262026-03-17T07:29:50.029Z-addresses = [{address = "192.168.1.30/24"}]
3272026-03-17T07:29:50.029Z+addresses = [{address = "10.151.2.174/24"} ]
3282026-03-17T07:29:50.029Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3292026-03-17T07:29:50.029Z port = "qsfp0"
3302026-03-17T07:29:50.029Z # The speed of this port.
3312026-03-17T07:29:50.029Za pkg/config-rss.toml 8K
3322026-03-17T07:29:50.029Za pkg/config.toml 6K
3332026-03-17T07:29:50.029Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3342026-03-17T07:29:50.029Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3352026-03-17T07:29:50.029Z+ TLS_NAME=recovery.sys.oxide.test
3362026-03-17T07:29:50.029Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3372026-03-17T07:29:50.162Z..............+.......+...+..+...+............+.+......+.....+.+...+..+...+.+.....+...+......+.+......+...+..+..........+..+..........+..+....+..+...+...............+.......+..+.......+......+.....+..........+..............+....+.........+..+...+...+..................+.+...+...........+.+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*......+...+....+..+.+............+...+..+..........+..+...+.+............+.....+...+.+......+..+...............+......+.......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...........+.....+...+...............+......+......+....+...............+..............+.........+.+.........+.....+...+...+.........+..........+...+..+.......+........+.......+...+........+..........+........+..........+..+....+.....................+............+...+...+.....+.....................+.+..+......+.......+............+..+....+..+.......+............+......+.........+...+...+.........+...........+.........+.+.....+...+..........+.........+...+.................+..........+......+..+...+......+..........+.....+....+.........+..+......+.........+.+.....+.........+...+...............+......+..........+.....+....+............+......+......................................+......+......+..................+.+...+...........+.+......+........+...............+.......+.........+..........................+.......+..+...+.+.........+.....+.+..............+...............+.+.........+.....+...+.......+........+.........+..........+.....+.........+.+......+..............+.+........+......+.+........+......+.+............+...+..............+...+.+......+........................+.....+....+.....+.+......+.....+.+..+..................+..........+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3382026-03-17T07:29:50.310Z.+...+..........+..+.+..+.......+...+......+..+.........+...+...............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.........+.+..+.......+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...................+...+............+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3392026-03-17T07:29:50.334Z-----
3402026-03-17T07:29:50.335Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3412026-03-17T07:29:50.359Za pkg/initial-tls-cert.pem 2K
3422026-03-17T07:29:50.359Za pkg/initial-tls-key.pem 4K
3432026-03-17T07:29:50.359Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3442026-03-17T07:29:50.359Z+ rmdir pkg
3452026-03-17T07:29:50.360Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3462026-03-17T07:29:50.360Z+ pfexec mkdir -p /usr/oxide
3472026-03-17T07:29:50.360Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3482026-03-17T07:29:51.022Z+ pfexec chmod +x /usr/oxide/oxide
3492026-03-17T07:29:51.047Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3502026-03-17T07:30:09.347Z+ OMICRON_NO_UNINSTALL=1
3512026-03-17T07:30:09.347Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3522026-03-17T07:30:09.371ZLogging to: /opt/oxide/work/out/LOG
3532026-03-17T07:30:10.184Z
3542026-03-17T07:30:10.209Zreal 0.832031972
3552026-03-17T07:30:10.209Zuser 0.174221517
3562026-03-17T07:30:10.209Zsys 4.814092208
3572026-03-17T07:30:10.209Ztrap 0.000008410
3582026-03-17T07:30:10.209Ztflt 0.000079070
3592026-03-17T07:30:10.209Zdflt 0.000231534
3602026-03-17T07:30:10.209Zkflt 0.000000000
3612026-03-17T07:30:10.209Zlock 1:15.006350588
3622026-03-17T07:30:10.209Zslp 1.012651715
3632026-03-17T07:30:10.209Zlat 0.021272042
3642026-03-17T07:30:10.209Zstop 0.072641652
3652026-03-17T07:30:10.209Z+ retry=0
3662026-03-17T07:30:10.209Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3672026-03-17T07:30:10.209Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3682026-03-17T07:30:10.209Z+ [[ 0 -gt 30 ]]
3692026-03-17T07:30:10.209Z+ sleep 1
3702026-03-17T07:30:11.202Z+ retry=1
3712026-03-17T07:30:11.202Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3722026-03-17T07:30:11.227Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3732026-03-17T07:30:11.227Z+ [[ 1 -gt 30 ]]
3742026-03-17T07:30:11.227Z+ sleep 1
3752026-03-17T07:30:12.214Z+ retry=2
3762026-03-17T07:30:12.214Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3772026-03-17T07:30:12.238Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3782026-03-17T07:30:12.238Z+ [[ 2 -gt 30 ]]
3792026-03-17T07:30:12.238Z+ sleep 1
 
3802026-03-17T07:30:12.363Zsock console login: Mar 17 07:30:11 sock svc.startd[8]: oxide/sled-agent:default failed repeatedly: transitioned to maintenance (see 'svcs -xv' for details)
3812026-03-17T07:30:12.401ZSUNW-MSG-ID: SMF-8000-YX, TYPE: defect, VER: 1, SEVERITY: major
3822026-03-17T07:30:12.446ZEVENT-TIME: Tue Mar 17 07:30:11 UTC 2026
3832026-03-17T07:30:12.518ZPLATFORM: R152-Z32-00, CSN: GKG8N7212A0003, HOSTNAME: sock
3842026-03-17T07:30:12.563ZSOURCE: software-diagnosis, REV: 0.1
3852026-03-17T07:30:12.608ZEVENT-ID: 162a8fd2-1663-409e-a2cc-b718046f63dd
3862026-03-17T07:30:12.654ZDESC: A service failed - the instance is restarting too quickly.
3872026-03-17T07:30:12.687ZRefer to http://illumos.org/msg/SMF-8000-YX for more information.
3882026-03-17T07:30:12.738ZAUTO-RESPONSE: The service has been placed into the maintenance state.
3892026-03-17T07:30:12.771ZIMPACT: svc:/oxide/sled-agent:default is unavailable.
3902026-03-17T07:30:12.805ZREC-ACTION: Run 'svcs -xv svc:/oxide/sled-agent:default' to determine the generic reason why the service failed, the location of any logfiles, and a list of other services impacted.
 
3912026-03-17T07:30:13.227Z+ retry=3
3922026-03-17T07:30:13.227Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3932026-03-17T07:30:13.252Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3942026-03-17T07:30:13.252Z+ [[ 3 -gt 30 ]]
3952026-03-17T07:30:13.252Z+ sleep 1
3962026-03-17T07:30:14.255Z+ retry=4
3972026-03-17T07:30:14.255Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3982026-03-17T07:30:14.281Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3992026-03-17T07:30:14.281Z+ [[ 4 -gt 30 ]]
4002026-03-17T07:30:14.281Z+ sleep 1
4012026-03-17T07:30:15.256Z+ retry=5
4022026-03-17T07:30:15.256Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4032026-03-17T07:30:15.281Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4042026-03-17T07:30:15.281Z+ [[ 5 -gt 30 ]]
4052026-03-17T07:30:15.281Z+ sleep 1
4062026-03-17T07:30:16.271Z+ retry=6
4072026-03-17T07:30:16.271Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4082026-03-17T07:30:16.295Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4092026-03-17T07:30:16.295Z+ [[ 6 -gt 30 ]]
4102026-03-17T07:30:16.295Z+ sleep 1
4112026-03-17T07:30:17.285Z+ retry=7
4122026-03-17T07:30:17.285Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4132026-03-17T07:30:17.311Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4142026-03-17T07:30:17.311Z+ [[ 7 -gt 30 ]]
4152026-03-17T07:30:17.311Z+ sleep 1
4162026-03-17T07:30:18.297Z+ retry=8
4172026-03-17T07:30:18.297Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4182026-03-17T07:30:18.322Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4192026-03-17T07:30:18.322Z+ [[ 8 -gt 30 ]]
4202026-03-17T07:30:18.322Z+ sleep 1
4212026-03-17T07:30:19.309Z+ retry=9
4222026-03-17T07:30:19.309Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4232026-03-17T07:30:19.334Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4242026-03-17T07:30:19.334Z+ [[ 9 -gt 30 ]]
4252026-03-17T07:30:19.334Z+ sleep 1
4262026-03-17T07:30:20.323Z+ retry=10
4272026-03-17T07:30:20.323Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4282026-03-17T07:30:20.352Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4292026-03-17T07:30:20.352Z+ [[ 10 -gt 30 ]]
4302026-03-17T07:30:20.352Z+ sleep 1
4312026-03-17T07:30:21.336Z+ retry=11
4322026-03-17T07:30:21.336Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4332026-03-17T07:30:21.360Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4342026-03-17T07:30:21.360Z+ [[ 11 -gt 30 ]]
4352026-03-17T07:30:21.360Z+ sleep 1
4362026-03-17T07:30:22.350Z+ retry=12
4372026-03-17T07:30:22.350Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4382026-03-17T07:30:22.375Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4392026-03-17T07:30:22.375Z+ [[ 12 -gt 30 ]]
4402026-03-17T07:30:22.375Z+ sleep 1
4412026-03-17T07:30:23.363Z+ retry=13
4422026-03-17T07:30:23.364Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4432026-03-17T07:30:23.388Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4442026-03-17T07:30:23.389Z+ [[ 13 -gt 30 ]]
4452026-03-17T07:30:23.389Z+ sleep 1
4462026-03-17T07:30:24.378Z+ retry=14
4472026-03-17T07:30:24.378Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4482026-03-17T07:30:24.403Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4492026-03-17T07:30:24.403Z+ [[ 14 -gt 30 ]]
4502026-03-17T07:30:24.403Z+ sleep 1
4512026-03-17T07:30:25.391Z+ retry=15
4522026-03-17T07:30:25.391Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4532026-03-17T07:30:25.416Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4542026-03-17T07:30:25.416Z+ [[ 15 -gt 30 ]]
4552026-03-17T07:30:25.416Z+ sleep 1
4562026-03-17T07:30:26.404Z+ retry=16
4572026-03-17T07:30:26.404Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4582026-03-17T07:30:26.428Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4592026-03-17T07:30:26.428Z+ [[ 16 -gt 30 ]]
4602026-03-17T07:30:26.428Z+ sleep 1
4612026-03-17T07:30:27.417Z+ retry=17
4622026-03-17T07:30:27.417Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4632026-03-17T07:30:27.442Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4642026-03-17T07:30:27.442Z+ [[ 17 -gt 30 ]]
4652026-03-17T07:30:27.442Z+ sleep 1
4662026-03-17T07:30:28.430Z+ retry=18
4672026-03-17T07:30:28.430Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4682026-03-17T07:30:28.454Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4692026-03-17T07:30:28.454Z+ [[ 18 -gt 30 ]]
4702026-03-17T07:30:28.454Z+ sleep 1
4712026-03-17T07:30:29.443Z+ retry=19
4722026-03-17T07:30:29.443Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4732026-03-17T07:30:29.469Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4742026-03-17T07:30:29.469Z+ [[ 19 -gt 30 ]]
4752026-03-17T07:30:29.469Z+ sleep 1
4762026-03-17T07:30:30.457Z+ retry=20
4772026-03-17T07:30:30.457Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4782026-03-17T07:30:30.482Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4792026-03-17T07:30:30.482Z+ [[ 20 -gt 30 ]]
4802026-03-17T07:30:30.482Z+ sleep 1
4812026-03-17T07:30:31.470Z+ retry=21
4822026-03-17T07:30:31.470Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4832026-03-17T07:30:31.495Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4842026-03-17T07:30:31.495Z+ [[ 21 -gt 30 ]]
4852026-03-17T07:30:31.495Z+ sleep 1
4862026-03-17T07:30:32.483Z+ retry=22
4872026-03-17T07:30:32.483Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4882026-03-17T07:30:32.507Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4892026-03-17T07:30:32.507Z+ [[ 22 -gt 30 ]]
4902026-03-17T07:30:32.507Z+ sleep 1
4912026-03-17T07:30:33.496Z+ retry=23
4922026-03-17T07:30:33.496Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4932026-03-17T07:30:33.521Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4942026-03-17T07:30:33.521Z+ [[ 23 -gt 30 ]]
4952026-03-17T07:30:33.521Z+ sleep 1
4962026-03-17T07:30:34.510Z+ retry=24
4972026-03-17T07:30:34.510Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4982026-03-17T07:30:34.535Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4992026-03-17T07:30:34.535Z+ [[ 24 -gt 30 ]]
5002026-03-17T07:30:34.535Z+ sleep 1
5012026-03-17T07:30:35.523Z+ retry=25
5022026-03-17T07:30:35.523Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5032026-03-17T07:30:35.548Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5042026-03-17T07:30:35.548Z+ [[ 25 -gt 30 ]]
5052026-03-17T07:30:35.548Z+ sleep 1
5062026-03-17T07:30:36.537Z+ retry=26
5072026-03-17T07:30:36.537Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5082026-03-17T07:30:36.561Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5092026-03-17T07:30:36.561Z+ [[ 26 -gt 30 ]]
5102026-03-17T07:30:36.561Z+ sleep 1
5112026-03-17T07:30:37.550Z+ retry=27
5122026-03-17T07:30:37.550Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5132026-03-17T07:30:37.574Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5142026-03-17T07:30:37.574Z+ [[ 27 -gt 30 ]]
5152026-03-17T07:30:37.574Z+ sleep 1
5162026-03-17T07:30:38.564Z+ retry=28
5172026-03-17T07:30:38.564Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5182026-03-17T07:30:38.589Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5192026-03-17T07:30:38.589Z+ [[ 28 -gt 30 ]]
5202026-03-17T07:30:38.589Z+ sleep 1
5212026-03-17T07:30:39.577Z+ retry=29
5222026-03-17T07:30:39.577Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5232026-03-17T07:30:39.602Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5242026-03-17T07:30:39.602Z+ [[ 29 -gt 30 ]]
5252026-03-17T07:30:39.602Z+ sleep 1
5262026-03-17T07:30:40.590Z+ retry=30
5272026-03-17T07:30:40.590Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5282026-03-17T07:30:40.615Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5292026-03-17T07:30:40.615Z+ [[ 30 -gt 30 ]]
5302026-03-17T07:30:40.615Z+ sleep 1
5312026-03-17T07:30:41.604Z+ retry=31
5322026-03-17T07:30:41.604Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5332026-03-17T07:30:41.628Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5342026-03-17T07:30:41.628Z+ [[ 31 -gt 30 ]]
5352026-03-17T07:30:41.628Z+ echo 'Failed to reach switch zone after 30 attempts'
5362026-03-17T07:30:41.628Z+ exit 1
5372026-03-17T07:30:41.629ZFailed to reach switch zone after 30 attempts
5382026-03-17T07:30:41.629Z+ _exit_trap
5392026-03-17T07:30:41.629Z+ local status=1
5402026-03-17T07:30:41.629Z+ set +o errexit
5412026-03-17T07:30:41.629Z+ [[ x != \x ]]
5422026-03-17T07:30:41.629Z+ pfexec svcadm -Z disable -s cron
5432026-03-17T07:30:41.669Z+ [[ 1 -eq 0 ]]
5442026-03-17T07:30:41.669Z+ set -o xtrace
5452026-03-17T07:30:41.669Z+ banner evidence
5462026-03-17T07:30:41.696Z
5472026-03-17T07:30:41.696Z ###### # # # ##### ###### # # #### ######
5482026-03-17T07:30:41.696Z # # # # # # # ## # # # #
5492026-03-17T07:30:41.696Z ##### # # # # # ##### # # # # #####
5502026-03-17T07:30:41.696Z # # # # # # # # # # # #
5512026-03-17T07:30:41.696Z # # # # # # # # ## # # #
5522026-03-17T07:30:41.696Z ###### ## # ##### ###### # # #### ######
5532026-03-17T07:30:41.696Z
5542026-03-17T07:30:41.696Z+ zoneadm list -civ
5552026-03-17T07:30:41.697Z ID NAME STATUS PATH BRAND IP
5562026-03-17T07:30:41.697Z 0 global running / ipkg shared
5572026-03-17T07:30:41.697Z 1 sidecar_softnpu running /sidecar/sidecar_softnpu omicron1 excl
5582026-03-17T07:30:41.697Z+ pfexec dladm show-phys -m
5592026-03-17T07:30:41.697ZLINK SLOT ADDRESS INUSE CLIENT
5602026-03-17T07:30:41.697Zigb0 primary 18:c0:4d:d:a0:2a yes igb0
5612026-03-17T07:30:41.697Zigb1 primary 18:c0:4d:d:a0:2b no --
5622026-03-17T07:30:41.697Z+ pfexec dladm show-link
5632026-03-17T07:30:41.697ZLINK CLASS MTU STATE BRIDGE OVER
5642026-03-17T07:30:41.697Zigb0 phys 1500 up -- --
5652026-03-17T07:30:41.697Zigb1 phys 1500 unknown -- --
5662026-03-17T07:30:41.697Znet0 simnet 9000 up -- sc0_0
5672026-03-17T07:30:41.697Zsc0_0 simnet 9000 up -- net0
5682026-03-17T07:30:41.697Znet1 simnet 9000 up -- sc1_0
5692026-03-17T07:30:41.697Zsc1_0 simnet 9000 up -- net1
5702026-03-17T07:30:41.697Zsc0_1 vnic 1500 up -- igb0
5712026-03-17T07:30:41.697Ztfportrear0_0 simnet 9000 up -- sidecar_shi0
5722026-03-17T07:30:41.697Zsidecar_shi0 simnet 9000 up -- tfportrear0_0
5732026-03-17T07:30:41.697Ztfportqsfp0_0 simnet 9000 up -- sidecar_shi1
5742026-03-17T07:30:41.697Zsidecar_shi1 simnet 9000 up -- tfportqsfp0_0
5752026-03-17T07:30:41.697Zbootstrap_stub0 etherstub 9000 up -- --
5762026-03-17T07:30:41.697Zbootstrap0 vnic 1500 up -- bootstrap_stub0
5772026-03-17T07:30:41.697Zunderlay_stub0 etherstub 9000 up -- --
5782026-03-17T07:30:41.698Zunderlay0 vnic 9000 up -- underlay_stub0
5792026-03-17T07:30:41.745Z+ pfexec dladm show-vnic
5802026-03-17T07:30:41.745ZLINK OVER SPEED MACADDRESS MACADDRTYPE VID
5812026-03-17T07:30:41.745Zsc0_1 igb0 1000 a8:e1:de:1:70:1d fixed 0
5822026-03-17T07:30:41.745Zbootstrap0 bootstrap_stub0 0 2:8:20:19:4b:9d random 0
5832026-03-17T07:30:41.745Zunderlay0 underlay_stub0 0 2:8:20:ef:6e:f1 random 0
5842026-03-17T07:30:41.745Z+ pfexec ipadm
5852026-03-17T07:30:41.745ZADDROBJ TYPE STATE ADDR
5862026-03-17T07:30:41.745Zlo0/v4 static ok 127.0.0.1/8
5872026-03-17T07:30:41.746Zigb0/dhcp dhcp ok 10.151.2.100/24
5882026-03-17T07:30:41.746Zlo0/v6 static ok ::1/128
5892026-03-17T07:30:41.746Znet0/ll addrconf ok fe80::f8d8:aaff:fe4a:f63f%net0/10
5902026-03-17T07:30:41.746Znet1/ll addrconf ok fe80::4ce0:d4ff:fe17:203e%net1/10
5912026-03-17T07:30:41.746Zbootstrap0/ll addrconf ok fe80::8:20ff:fe19:4b9d%bootstrap0/10
5922026-03-17T07:30:41.746Zbootstrap0/bootstrap6 static ok fdb0:18c0:4d0d:a02a::1/64
5932026-03-17T07:30:41.746Z+ pfexec netstat -rncva
5942026-03-17T07:30:41.746Z
5952026-03-17T07:30:41.746ZIRE Table: IPv4
5962026-03-17T07:30:41.746Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
5972026-03-17T07:30:41.746Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
5982026-03-17T07:30:41.746Z0.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
5992026-03-17T07:30:41.746Zdefault 0.0.0.0 10.151.2.1 igb0 1500 9 UG 1694048 0
6002026-03-17T07:30:41.746Z10.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6012026-03-17T07:30:41.746Z10.151.2.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6022026-03-17T07:30:41.746Z10.151.2.0/24 255.255.255.0 10.151.2.100 igb0 1500 3 U 0 0
6032026-03-17T07:30:41.746Z10.151.2.1/32 255.255.255.255 10.151.2.100 igb0 1500 3 UC 0 0
6042026-03-17T07:30:41.746Z10.151.2.100 255.255.255.255 -- igb0 1500 2 UHL 2 3387008
6052026-03-17T07:30:41.746Z10.151.2.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6062026-03-17T07:30:41.746Z10.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6072026-03-17T07:30:41.746Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 56 56
6082026-03-17T07:30:41.746Z255.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6092026-03-17T07:30:41.746Z
6102026-03-17T07:30:41.746ZIRE Table: IPv6
6112026-03-17T07:30:41.746Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
6122026-03-17T07:30:41.746Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
6132026-03-17T07:30:41.746Z::1 ::1 lo0 8252 2 UH 28 28
6142026-03-17T07:30:41.746Zfdb0:18c0:4d0d:a02a::1 -- bootstrap0 1500 2 UHL 0 0
6152026-03-17T07:30:41.746Zfe80::4ce0:d4ff:fe17:203e -- net1 9000 2 UHL 0 0
6162026-03-17T07:30:41.746Zfe80::f8d8:aaff:fe4a:f63f -- net0 9000 2 UHL 0 0
6172026-03-17T07:30:41.746Zfe80::8:20ff:fe19:4b9d -- bootstrap0 1500 2 UHL 0 0
6182026-03-17T07:30:41.746Zfdb0:18c0:4d0d:a02a::/64 fdb0:18c0:4d0d:a02a::1 bootstrap0 1500 2 U 0 0
6192026-03-17T07:30:41.746Zfe80::/10 fe80::8:20ff:fe19:4b9d bootstrap0 1500 2 U 0 0
6202026-03-17T07:30:41.746Zfe80::/10 fe80::4ce0:d4ff:fe17:203e net1 9000 2 U 0 0
6212026-03-17T07:30:41.746Zfe80::/10 fe80::f8d8:aaff:fe4a:f63f net0 9000 2 U 0 0
6222026-03-17T07:30:41.746Z+ pfexec netstat -anu
6232026-03-17T07:30:41.746Z
6242026-03-17T07:30:41.746ZUDP: IPv4
6252026-03-17T07:30:41.746Z Local Address Remote Address User Pid Command State
6262026-03-17T07:30:41.746Z-------------------- -------------------- -------- ------ -------------- ----------
6272026-03-17T07:30:41.747Z *.* root 71 in.mpathd Unbound
6282026-03-17T07:30:41.747Z *.* root 71 in.mpathd Unbound
6292026-03-17T07:30:41.747Z *.68 root 298 dhcpagent Idle
6302026-03-17T07:30:41.747Z *.546 root 298 dhcpagent Idle
6312026-03-17T07:30:41.747Z10.151.2.100.68 root 298 dhcpagent Idle
6322026-03-17T07:30:41.747Z *.111 daemon 712 rpcbind Idle
6332026-03-17T07:30:41.747Z *.* daemon 712 rpcbind Unbound
6342026-03-17T07:30:41.747Z *.64474 daemon 712 rpcbind Idle
6352026-03-17T07:30:41.747Z *.111 daemon 712 rpcbind Idle
6362026-03-17T07:30:41.747Z *.* daemon 712 rpcbind Unbound
6372026-03-17T07:30:41.747Z *.59823 daemon 712 rpcbind Idle
6382026-03-17T07:30:41.747Z *.* root 722 in.ndpd Unbound
6392026-03-17T07:30:41.747Z *.* root 867 rcm_daemon Unbound
6402026-03-17T07:30:41.747Z *.* root 867 rcm_daemon Unbound
6412026-03-17T07:30:41.747Z
6422026-03-17T07:30:41.747ZUDP: IPv6
6432026-03-17T07:30:41.747Z Local Address Remote Address User Pid Command State If
6442026-03-17T07:30:41.747Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
6452026-03-17T07:30:41.747Z *.* root 71 in.mpathd Unbound
6462026-03-17T07:30:41.747Z *.546 root 298 dhcpagent Idle
6472026-03-17T07:30:41.747Z *.111 daemon 712 rpcbind Idle
6482026-03-17T07:30:41.747Z *.* daemon 712 rpcbind Unbound
6492026-03-17T07:30:41.747Z *.64474 daemon 712 rpcbind Idle
6502026-03-17T07:30:41.747Z *.* root 722 in.ndpd Unbound
6512026-03-17T07:30:41.747Z *.* root 867 rcm_daemon Unbound
6522026-03-17T07:30:41.747Zff02::dd.3549 root 1575 ddmd Idle net1
6532026-03-17T07:30:41.747Zff02::dd.3549 root 1575 ddmd Idle net0
6542026-03-17T07:30:41.747Zfe80::4ce0:d4ff:fe17:203e.3549 root 1575 ddmd Idle net1
6552026-03-17T07:30:41.747Zfe80::f8d8:aaff:fe4a:f63f.3549 root 1575 ddmd Idle net0
6562026-03-17T07:30:41.747Z
6572026-03-17T07:30:41.747ZTCP: IPv4
6582026-03-17T07:30:41.747Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State
6592026-03-17T07:30:41.747Z-------------------- -------------------- -------- ------ -------------- ------ ------ ------ ------ -----------
6602026-03-17T07:30:41.747Z127.0.0.1.5999 *.* root 71 in.mpathd 0 0 128000 0 LISTEN
6612026-03-17T07:30:41.747Z127.0.0.1.4999 *.* root 298 dhcpagent 0 0 128000 0 LISTEN
6622026-03-17T07:30:41.747Z *.111 *.* daemon 712 rpcbind 0 0 128000 0 LISTEN
6632026-03-17T07:30:41.747Z *.* *.* daemon 712 rpcbind 0 0 128000 0 IDLE
6642026-03-17T07:30:41.747Z *.111 *.* daemon 712 rpcbind 0 0 128000 0 LISTEN
6652026-03-17T07:30:41.747Z *.* *.* daemon 712 rpcbind 0 0 128000 0 IDLE
6662026-03-17T07:30:41.748Z10.151.2.100.61928 54.187.123.254.443 root 802 agent 1049792 0 128872 0 ESTABLISHED
6672026-03-17T07:30:41.748Z10.151.2.100.55398 54.187.123.254.443 root 802 agent 1049792 0 128872 0 ESTABLISHED
6682026-03-17T07:30:41.748Z10.151.2.100.64087 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6692026-03-17T07:30:41.748Z10.151.2.100.45804 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6702026-03-17T07:30:41.748Z10.151.2.100.51957 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6712026-03-17T07:30:41.748Z10.151.2.100.49744 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
6722026-03-17T07:30:41.748Z10.151.2.100.43227 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
6732026-03-17T07:30:41.748Z
6742026-03-17T07:30:41.748ZTCP: IPv6
6752026-03-17T07:30:41.748Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State If
6762026-03-17T07:30:41.748Z--------------------------------- --------------------------------- -------- ------ -------------- ------ ------ ------ ------ ----------- -----
6772026-03-17T07:30:41.748Z::1.5999 *.* root 71 in.mpathd 0 0 128000 0 LISTEN
6782026-03-17T07:30:41.748Z *.111 *.* daemon 712 rpcbind 0 0 128000 0 LISTEN
6792026-03-17T07:30:41.773Z *.* *.* daemon 712 rpcbind 0 0 128000 0 IDLE
6802026-03-17T07:30:41.773Z::1.8000 *.* root 1575 ddmd 0 0 128000 0 LISTEN
6812026-03-17T07:30:41.773Z
6822026-03-17T07:30:41.773ZActive UNIX domain sockets
6832026-03-17T07:30:41.773ZAddress Type User Pid Command Local Address Remote Address
6842026-03-17T07:30:41.773Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
6852026-03-17T07:30:41.774Zfffffe85b75677a0 stream-ord root 1575 ddmd
6862026-03-17T07:30:41.774Zfffffe85b75677a0 stream-ord root 1575 ddmd
6872026-03-17T07:30:41.774Zfffffe85b7567b40 stream-ord root 1575 ddmd
6882026-03-17T07:30:41.774Zfffffe859f305030 stream-ord root 888 zoneadmd /var/run/zones/sidecar_softnpu.console_
6892026-03-17T07:30:41.774Zfffffe8569cfdaf8 stream-ord root 802 agent /var/run/buildomat.sock
6902026-03-17T07:30:41.774Zfffffe8586e4f010 stream-ord root 802 agent
6912026-03-17T07:30:41.774Zfffffe8586e4f010 stream-ord root 802 agent
6922026-03-17T07:30:41.774Zfffffe8586e4f3b0 stream-ord root 802 agent
6932026-03-17T07:30:41.774Zfffffe85842533a8 dgram root 722 in.ndpd /var/run/in.ndpd_mib
6942026-03-17T07:30:41.774Zfffffe8584253748 stream-ord root 722 in.ndpd /var/run/in.ndpd_ipadm
6952026-03-17T07:30:41.774Zfffffe8584253ae8 stream-ord root 757 inetd /var/run/.inetd.uds
6962026-03-17T07:30:41.774Z+ pfexec arp -an
6972026-03-17T07:30:41.774ZNet to Media Table: IPv4
6982026-03-17T07:30:41.774ZDevice IP Address Mask Flags Phys Addr
6992026-03-17T07:30:41.774Z------ -------------------- --------------- -------- ---------------
7002026-03-17T07:30:41.774Zigb0 10.151.2.1 255.255.255.255 aa:00:04:00:ca:fe
7012026-03-17T07:30:41.774Zigb0 10.151.2.100 255.255.255.255 SPLA 18:c0:4d:0d:a0:2a
7022026-03-17T07:30:41.774Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
7032026-03-17T07:30:41.774Zlocal v6:
7042026-03-17T07:30:41.774Zlocal v4:
7052026-03-17T07:30:41.774Zrouter v6_idx:
7062026-03-17T07:30:41.774Zrouter v6_routes:
7072026-03-17T07:30:41.774Zrouter v4_idx:
7082026-03-17T07:30:41.774Zrouter v4_routes:
7092026-03-17T07:30:41.774Zresolver v4:
7102026-03-17T07:30:41.774Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
7112026-03-17T07:30:41.774Zresolver v6:
7122026-03-17T07:30:41.774Znat v4:
7132026-03-17T07:30:41.774Znat_v6:
7142026-03-17T07:30:41.774Zattached subnet v4:
7152026-03-17T07:30:41.774Zattached subnet v6:
7162026-03-17T07:30:41.774Zport_mac:
7172026-03-17T07:30:41.774Zproxy arp:
7182026-03-17T07:30:41.774Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
7192026-03-17T07:30:41.774Z+ pfexec /opt/oxide/opte/bin/opteadm list-ports
7202026-03-17T07:30:41.774ZLINK MAC ADDRESS IPv4 ADDRESS EPHEMERAL IPv4 FLOATING IPv4 IPv6 ADDRESS EXTERNAL IPv6 FLOATING IPv6 STATE
7212026-03-17T07:30:41.774Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2b
7222026-03-17T07:30:41.774ZVirtual to Boundary Mappings
7232026-03-17T07:30:41.774Z======================================================================
7242026-03-17T07:30:41.774Z
7252026-03-17T07:30:41.775ZIPv4 mappings
7262026-03-17T07:30:41.775Z----------------------------------------------------------------------
7272026-03-17T07:30:41.775ZTUNNELED PREFIX BOUNDARY IP VNI
7282026-03-17T07:30:41.775Z
7292026-03-17T07:30:41.775ZIPv6 mappings
7302026-03-17T07:30:41.775Z----------------------------------------------------------------------
7312026-03-17T07:30:41.775ZTUNNELED PREFIX BOUNDARY IP VNI
7322026-03-17T07:30:41.775Z
7332026-03-17T07:30:41.775Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2p
7342026-03-17T07:30:41.775ZVirtual to Physical Mappings
7352026-03-17T07:30:41.775Z======================================================================
7362026-03-17T07:30:41.775Z+ z_swadm link ls
7372026-03-17T07:30:41.775Z+ echo '== swadm link' ls
7382026-03-17T07:30:41.775Z== swadm link ls
7392026-03-17T07:30:41.775Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm link ls
7402026-03-17T07:30:41.775Zzlogin: zone 'oxz_switch' unknown
7412026-03-17T07:30:41.775Z+ z_swadm addr list
7422026-03-17T07:30:41.775Z+ echo '== swadm addr' list
7432026-03-17T07:30:41.775Z== swadm addr list
7442026-03-17T07:30:41.775Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm addr list
7452026-03-17T07:30:41.800Zzlogin: zone 'oxz_switch' unknown
7462026-03-17T07:30:41.800Z+ z_swadm route list
7472026-03-17T07:30:41.800Z+ echo '== swadm route' list
7482026-03-17T07:30:41.800Z== swadm route list
7492026-03-17T07:30:41.800Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm route list
7502026-03-17T07:30:41.800Zzlogin: zone 'oxz_switch' unknown
7512026-03-17T07:30:41.800Z+ z_swadm arp list
7522026-03-17T07:30:41.800Z+ echo '== swadm arp' list
7532026-03-17T07:30:41.800Z== swadm arp list
7542026-03-17T07:30:41.800Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm arp list
7552026-03-17T07:30:41.800Zzlogin: zone 'oxz_switch' unknown
7562026-03-17T07:30:41.800Z+ z_swadm nat list
7572026-03-17T07:30:41.800Z+ echo '== swadm nat' list
7582026-03-17T07:30:41.800Z== swadm nat list
7592026-03-17T07:30:41.800Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm nat list
7602026-03-17T07:30:41.800Zzlogin: zone 'oxz_switch' unknown
7612026-03-17T07:30:41.800Z++ pfexec /opt/oxide/opte/bin/opteadm list-ports
7622026-03-17T07:30:41.800Z++ tail +2
7632026-03-17T07:30:41.800Z++ awk '{ print $1; }'
7642026-03-17T07:30:41.801Z+ PORTS=
7652026-03-17T07:30:41.801Z+ mkdir -p /tmp/kstat
7662026-03-17T07:30:41.826Z+ pfexec kstat -p xde:
7672026-03-17T07:30:41.826Z+ pfexec zfs list
7682026-03-17T07:30:41.826ZNAME USED AVAIL REFER MOUNTPOINT
7692026-03-17T07:30:41.826Zrpool 2.34G 4.92G 24K /rpool
7702026-03-17T07:30:41.826Zrpool/home 25.5K 4.92G 25.5K /home
7712026-03-17T07:30:41.826Zrpool/input 2.27G 4.92G 2.27G /input
7722026-03-17T07:30:41.826Zrpool/sidecar 75.8M 4.92G 24K /sidecar
7732026-03-17T07:30:41.826Zrpool/sidecar/sidecar_softnpu 75.7M 4.92G 75.7M /sidecar/sidecar_softnpu
7742026-03-17T07:30:41.826Zrpool/work 24K 4.92G 24K /work
7752026-03-17T07:30:41.826Zrpool/zone 24K 4.92G 24K /zone
7762026-03-17T07:30:41.826Zscratch 118K 1.80T 25K /scratch
7772026-03-17T07:30:41.826Z+ pfexec zpool list
7782026-03-17T07:30:41.826ZNAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
7792026-03-17T07:30:41.826Zrpool 7.50G 2.34G 5.16G - - 0% 31% 1.00x ONLINE -
7802026-03-17T07:30:41.826Zscratch 1.86T 118K 1.86T - - 0% 0% 1.00x ONLINE -
7812026-03-17T07:30:41.826Z+ pfexec fmdump -eVp
7822026-03-17T07:30:41.826ZTIME CLASS
7832026-03-17T07:30:41.826Z+ pfexec ptree -z global
7842026-03-17T07:30:41.851Z8 /lib/svc/bin/svc.startd
7852026-03-17T07:30:41.851Z 639 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
7862026-03-17T07:30:41.851Z 662 /usr/lib/saf/sac -t 300
7872026-03-17T07:30:41.851Z 676 /usr/lib/saf/ttymon
7882026-03-17T07:30:41.851Z 801 /usr/bin/ctrun -l child -o noorphan,regent /opt/buildomat/lib/agent ru
7892026-03-17T07:30:41.851Z 802 /opt/buildomat/lib/agent run
7902026-03-17T07:30:41.851Z 816 /bin/bash /tmp/HIiAtMyXCE9IGdHM.sh
7912026-03-17T07:30:41.851Z 1782 ptree -z global
7922026-03-17T07:30:41.851Z10 /lib/svc/bin/svc.configd
7932026-03-17T07:30:41.852Z39 /lib/inet/ipmgmtd
7942026-03-17T07:30:41.852Z42 /lib/inet/netcfgd
7952026-03-17T07:30:41.852Z47 /sbin/dlmgmtd
7962026-03-17T07:30:41.852Z71 /lib/inet/in.mpathd
7972026-03-17T07:30:41.852Z94 /usr/lib/pfexecd
7982026-03-17T07:30:41.852Z149 /usr/lib/sysevent/syseventd
7992026-03-17T07:30:41.852Z150 /usr/lib/zones/zonestatd
8002026-03-17T07:30:41.852Z154 devfsadmd
8012026-03-17T07:30:41.852Z298 /sbin/dhcpagent
8022026-03-17T07:30:41.852Z565 /usr/lib/picl/picld
8032026-03-17T07:30:41.852Z597 /usr/sbin/nscd
8042026-03-17T07:30:41.852Z638 /usr/lib/utmpd
8052026-03-17T07:30:41.852Z681 /usr/sbin/syslogd
8062026-03-17T07:30:41.852Z685 /usr/lib/hotplugd
8072026-03-17T07:30:41.852Z712 /usr/sbin/rpcbind
8082026-03-17T07:30:41.852Z722 /usr/lib/inet/in.ndpd
8092026-03-17T07:30:41.852Z723 /usr/lib/fm/fmd/fmd
8102026-03-17T07:30:41.852Z757 /usr/lib/inet/inetd start
8112026-03-17T07:30:41.852Z782 /usr/perl5/bin/perl /usr/lib/intrd
8122026-03-17T07:30:41.852Z867 /usr/lib/rcm/rcm_daemon
8132026-03-17T07:30:41.852Z888 zoneadmd -z sidecar_softnpu
8142026-03-17T07:30:41.852Z1573 ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_scrip
8152026-03-17T07:30:41.852Z 1575 /opt/oxide/mg-ddm/ddmd --with-stats --admin-port 8000 --admin-addr ::1
8162026-03-17T07:30:41.852Z+ pfexec svcs -xv
8172026-03-17T07:30:41.897Zsvc:/oxide/sled-agent:default (Oxide Sled Agent)
8182026-03-17T07:30:41.898Z State: maintenance since March 17, 2026 at 07:30:11 AM UTC
8192026-03-17T07:30:41.898ZReason: Restarting too quickly.
8202026-03-17T07:30:41.898Z See: http://illumos.org/msg/SMF-8000-L5
8212026-03-17T07:30:41.898Z See: /var/svc/log/oxide-sled-agent:default.log
8222026-03-17T07:30:41.898ZImpact: This service is not running.
8232026-03-17T07:30:41.922Z++ zoneadm list -n
8242026-03-17T07:30:41.922Z+ for z in $(zoneadm list -n)
8252026-03-17T07:30:41.922Z+ banner sidecar_softnpu
8262026-03-17T07:30:41.922Z
8272026-03-17T07:30:41.922Z #### # ##### ###### #### ## ##### #### ####
8282026-03-17T07:30:41.922Z # # # # # # # # # # # # # #
8292026-03-17T07:30:41.922Z #### # # # ##### # # # # # #### # #
8302026-03-17T07:30:41.922Z # # # # # # ###### ##### # # #
8312026-03-17T07:30:41.922Z # # # # # # # # # # # # # # # #
8322026-03-17T07:30:41.922Z #### # ##### ###### #### # # # # ####### #### ####
8332026-03-17T07:30:41.922Z
8342026-03-17T07:30:41.922Z+ pfexec svcs -xv -z sidecar_softnpu
8352026-03-17T07:30:41.963Z+ pfexec ptree -z sidecar_softnpu
8362026-03-17T07:30:41.988Z891 zsched
8372026-03-17T07:30:41.988Z 908 /sbin/init
8382026-03-17T07:30:41.988Z913 /lib/svc/bin/svc.startd
8392026-03-17T07:30:41.988Z 1442 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
8402026-03-17T07:30:41.988Z915 /lib/svc/bin/svc.configd
8412026-03-17T07:30:41.988Z961 /lib/inet/ipmgmtd
8422026-03-17T07:30:41.988Z963 /lib/inet/netcfgd
8432026-03-17T07:30:41.988Z1140 /usr/lib/fm/fmd/fmd
8442026-03-17T07:30:41.988Z1179 /usr/lib/pfexecd
8452026-03-17T07:30:41.988Z1289 /softnpu/softnpu --uds-path /softnpu /softnpu/softnpu.toml
8462026-03-17T07:30:41.988Z1391 /usr/sbin/nscd
8472026-03-17T07:30:41.988Z1441 /usr/lib/utmpd
8482026-03-17T07:30:41.988Z1465 /usr/sbin/syslogd
8492026-03-17T07:30:41.988Z1514 /usr/lib/inet/in.ndpd
8502026-03-17T07:30:41.988Z+ pfexec zlogin sidecar_softnpu ipadm
8512026-03-17T07:30:41.988ZADDROBJ TYPE STATE ADDR
8522026-03-17T07:30:41.988Zlo0/v4 static ok 127.0.0.1/8
8532026-03-17T07:30:41.988Zlo0/v6 static ok ::1/128
8542026-03-17T07:30:42.013Z+ pfexec zlogin sidecar_softnpu netstat -rncva
8552026-03-17T07:30:42.013Z
8562026-03-17T07:30:42.013ZIRE Table: IPv4
8572026-03-17T07:30:42.013Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
8582026-03-17T07:30:42.013Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
8592026-03-17T07:30:42.013Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 0 0
8602026-03-17T07:30:42.014Z
8612026-03-17T07:30:42.014ZIRE Table: IPv6
8622026-03-17T07:30:42.014Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
8632026-03-17T07:30:42.014Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
8642026-03-17T07:30:42.014Z::1 ::1 lo0 8252 2 UH 0 0
8652026-03-17T07:30:42.014Z+ pfexec zlogin sidecar_softnpu netstat -anu
8662026-03-17T07:30:42.038Z
8672026-03-17T07:30:42.038ZUDP: IPv4
8682026-03-17T07:30:42.038Z Local Address Remote Address User Pid Command State
8692026-03-17T07:30:42.038Z-------------------- -------------------- -------- ------ -------------- ----------
8702026-03-17T07:30:42.038Z *.* root 1514 in.ndpd Unbound
8712026-03-17T07:30:42.038Z
8722026-03-17T07:30:42.038ZUDP: IPv6
8732026-03-17T07:30:42.038Z Local Address Remote Address User Pid Command State If
8742026-03-17T07:30:42.038Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
8752026-03-17T07:30:42.038Z *.* root 1514 in.ndpd Unbound
8762026-03-17T07:30:42.039Z
8772026-03-17T07:30:42.039ZActive UNIX domain sockets
8782026-03-17T07:30:42.039ZAddress Type User Pid Command Local Address Remote Address
8792026-03-17T07:30:42.039Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
8802026-03-17T07:30:42.039Zfffffe85ae3d93e8 dgram root 1514 in.ndpd /var/run/in.ndpd_mib
8812026-03-17T07:30:42.039Zfffffe85ae3d9788 stream-ord root 1514 in.ndpd /var/run/in.ndpd_ipadm
8822026-03-17T07:30:42.039Zfffffe85b1162778 stream-ord root 1289 softnpu
8832026-03-17T07:30:42.039Zfffffe85b1162778 stream-ord root 1289 softnpu
8842026-03-17T07:30:42.039Zfffffe85b1162b18 stream-ord root 1289 softnpu
8852026-03-17T07:30:42.039Zfffffe8584253008 dgram root 1289 softnpu /softnpu/server
8862026-03-17T07:30:42.039Z+ pfexec zlogin sidecar_softnpu arp -an
8872026-03-17T07:30:42.063Z++ zoneadm list -n
8882026-03-17T07:30:42.063Z++ grep oxz_ntp
8892026-03-17T07:30:42.063Z+ pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log
8902026-03-17T07:30:42.088ZConfig {
8912026-03-17T07:30:42.088Z p4_program: "/softnpu/asic_program.so",
8922026-03-17T07:30:42.088Z ports: [
8932026-03-17T07:30:42.088Z Port {
8942026-03-17T07:30:42.088Z sidecar: "sc0_0",
8952026-03-17T07:30:42.088Z scrimlet: "sidecar_shi0",
8962026-03-17T07:30:42.088Z mtu: 9000,
8972026-03-17T07:30:42.088Z },
8982026-03-17T07:30:42.088Z Port {
8992026-03-17T07:30:42.088Z sidecar: "sc0_1",
9002026-03-17T07:30:42.088Z scrimlet: "sidecar_shi1",
9012026-03-17T07:30:42.088Z mtu: 9000,
9022026-03-17T07:30:42.088Z },
9032026-03-17T07:30:42.088Z ],
9042026-03-17T07:30:42.088Z}
9052026-03-17T07:30:42.088Z+ exit 1
9062026-03-17T07:30:42.088Zprocess exited: duration 88178 ms, exit code 1
 
9072026-03-17T07:30:42.139Zfound 3 output files
9082026-03-17T07:30:42.139Zuploading: /var/svc/log/oxide-mg-ddm:default.log (4406 bytes)
9092026-03-17T07:30:43.239Zuploaded: /var/svc/log/oxide-mg-ddm:default.log
9102026-03-17T07:30:43.239Zuploading: /var/svc/log/oxide-sled-agent:default.log (15377 bytes)
9112026-03-17T07:30:44.361Zuploaded: /var/svc/log/oxide-sled-agent:default.log
9122026-03-17T07:30:44.362Zuploading: /tmp/kstat/xde.kstat (890 bytes)
9132026-03-17T07:30:45.461Zuploaded: /tmp/kstat/xde.kstat