01KKX538EWPF8F7NVQ40DPPDNQ: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.39",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KKX5418H3ENAPFPT7B3JPS9X

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-03-17T06:06:21.582Zjob dependencies complete; ready to run (waiting for 22 m 56 s)
22026-03-17T06:06:21.585Zjob assigned to worker 01KKX54YSDBR8YYMENSZYWCBK3 [factory lab, sock/16329] (queued for 0 s)
32026-03-17T06:06:21.818Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-03-17T06:06:21.858Zxcvsh0tfvd xcvsh0tfvd xcvsh0tfvd xcvsh0tfvd xcvsh0tfvd xcvsh0tfvd xcvsh0tfvd xcvsh0tfvd
52026-03-17T06:06:21.891Zhttp://172.20.0.5:9969/ipxe/sock... ok
62026-03-17T06:06:21.924Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
72026-03-17T06:06:21.958Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
82026-03-17T06:06:21.992Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
92026-03-17T06:06:22.025Zhttp://172.20.0.5:9969/postboot/sock... ok
102026-03-17T06:06:22.059ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
112026-03-17T06:06:22.093ZWARNING: cpu0: failed to update microcode from version 0xa001133 to 0xa0011de
122026-03-17T06:06:22.126ZWARNING: cpu0: using older fallback microcode; update the system firmware
132026-03-17T06:06:22.160Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d5
142026-03-17T06:06:22.193ZOxide Helios Version helios-2.0.23743 64-bit
152026-03-17T06:06:22.228ZConfiguring devices.
162026-03-17T06:06:22.260Z##################################################
172026-03-17T06:06:22.294Z#### ############# ##
182026-03-17T06:06:22.327Z### ### ############ ##
192026-03-17T06:06:22.361Z## ### # ## ### ## Oxide Computer Company ##
202026-03-17T06:06:22.401Z## ## ## ### # ### ##
212026-03-17T06:06:22.457Z## # ### #### #### This Station Under ##
222026-03-17T06:06:22.529Z### ### #### # ### Computer Control ##
232026-03-17T06:06:22.596Z#### #### ### ## ##
242026-03-17T06:06:22.629Z##################################################
252026-03-17T06:06:22.662ZHostname: sock
262026-03-17T06:06:22.696ZLoading smf(7) service descriptions:
272026-03-17T06:06:23.893Zdownloading input: /input/package/work/package.tar.gz
282026-03-17T06:13:25.353Zdownloaded input: /input/package/work/package.tar.gz
 
292026-03-17T06:13:25.378Zstarting task 0: "setup"
302026-03-17T06:13:25.378Z++ uname -s
312026-03-17T06:13:25.378Z+ kern=SunOS
322026-03-17T06:13:25.378Z+ build_user=build
332026-03-17T06:13:25.378Z+ build_uid=12345
342026-03-17T06:13:25.378Z+ work_dir=/work
352026-03-17T06:13:25.378Z+ input_dir=/input
362026-03-17T06:13:25.378Z+ [[ 0 == 12345 ]]
372026-03-17T06:13:25.378Z+ case "$kern" in
382026-03-17T06:13:25.378Z+ groupadd -g 12345 build
392026-03-17T06:13:25.378Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
402026-03-17T06:13:27.372Z+ zfs create -o mountpoint=/work rpool/work
412026-03-17T06:13:27.396Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
422026-03-17T06:13:27.396Z+ home_fs=zfs
432026-03-17T06:13:27.396Z+ [[ zfs == autofs ]]
442026-03-17T06:13:27.396Z+ mkdir -p /home/build
452026-03-17T06:13:27.420Z+ chown build:build /home/build /work
462026-03-17T06:13:29.401Z+ chmod 0700 /home/build /work
472026-03-17T06:13:29.426Zprocess exited: duration 4051 ms, exit code 0
 
482026-03-17T06:13:29.473Zstarting task 1: "authentication"
492026-03-17T06:13:29.518Zprocess exited: duration 44 ms, exit code 0
 
502026-03-17T06:13:29.567Zstarting task 2: "build"
512026-03-17T06:13:29.591Z+ trap _exit_trap EXIT
522026-03-17T06:13:29.591Z+ OPTE_COMMIT=
532026-03-17T06:13:29.592Z+ [[ x != \x ]]
542026-03-17T06:13:29.592Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
552026-03-17T06:13:29.686Zsecpolicy_fs_linkdir: 0x55 = 0x55
562026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
572026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
582026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+3: 0 = 0xe5
592026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+4: 0 = 0xb8
602026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+5: 0 = 0x1
612026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
622026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
632026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
642026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
652026-03-17T06:13:29.686Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
662026-03-17T06:13:29.710Z+ [[ -d /opt/oxide ]]
672026-03-17T06:13:29.710Z+ pfexec mv /opt/oxide /opt/oxide-underneath
682026-03-17T06:13:29.711Z+ pfexec mkdir /opt/oxide
692026-03-17T06:13:29.711Z+ pfexec mount -F tmpfs -O swap /opt/oxide
702026-03-17T06:13:29.711Z+ [[ -d /opt/oxide-underneath ]]
712026-03-17T06:13:29.711Z+ cd /opt/oxide-underneath
722026-03-17T06:13:29.711Z+ pfexec tar ceEp@/f - .
732026-03-17T06:13:29.711Z+ cd /opt/oxide
742026-03-17T06:13:29.711Z+ pfexec tar xveEp@/f -
752026-03-17T06:13:29.753Zx ., 0 bytes, 0 tape blocks
762026-03-17T06:13:29.753Zx ./opte, 0 bytes, 0 tape blocks
772026-03-17T06:13:29.753Zx ./opte/bin, 0 bytes, 0 tape blocks
782026-03-17T06:13:29.753Zx ./opte/bin/opteadm, 22792392 bytes, 44517 tape blocks
792026-03-17T06:13:29.777Z+ pfexec rm -rf /opt/oxide-underneath
802026-03-17T06:13:29.777Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
812026-03-17T06:13:29.802Z+ pfexec mkdir /opt/oxide/work
822026-03-17T06:13:29.802Z+ pfexec chown build:build /opt/oxide/work
832026-03-17T06:13:29.802Z+ cd /opt/oxide/work
842026-03-17T06:13:29.802Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
852026-03-17T06:13:51.344ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
862026-03-17T06:13:51.344Zx .github/buildomat/ci-env.sh, 196 bytes, 1 tape blocks
872026-03-17T06:13:51.344Zx out/target/test, 107 bytes, 1 tape blocks
882026-03-17T06:13:51.344Zx out/npuzone/npuzone, 28059200 bytes, 54804 tape blocks
892026-03-17T06:13:51.344Zx package-manifest.toml, 34895 bytes, 69 tape blocks
902026-03-17T06:13:51.344Zx smf/sled-agent/non-gimlet/config.toml, 5328 bytes, 11 tape blocks
912026-03-17T06:13:51.344Zx target/release/omicron-package, 41908560 bytes, 81853 tape blocks
922026-03-17T06:13:51.344Zx target/release/xtask, 11699728 bytes, 22852 tape blocks
932026-03-17T06:13:51.344Zx target/debug/bootstrap, 343921080 bytes, 671721 tape blocks
942026-03-17T06:13:51.344Zx tests/bootstrap-ce927ede589904ba, 32061232 bytes, 62620 tape blocks
952026-03-17T06:13:51.344Zx tests/commtest-ed680c524300f199, 33172184 bytes, 64790 tape blocks
962026-03-17T06:13:51.344Zx tests/dhcp_server-530ecc14a0c4d124, 32061248 bytes, 62620 tape blocks
972026-03-17T06:13:51.344Zx tests/end_to_end_tests-3148bac0dcf3127f, 386821536 bytes, 755511 tape blocks
982026-03-17T06:13:51.344Zx out/clickhouse.tar.gz, 325126989 bytes, 635014 tape blocks
992026-03-17T06:13:51.344Zx out/clickhouse_keeper.tar.gz, 308717586 bytes, 602965 tape blocks
1002026-03-17T06:13:51.344Zx out/clickhouse_server.tar.gz, 325125809 bytes, 635012 tape blocks
1012026-03-17T06:13:51.344Zx out/cockroachdb.tar.gz, 169974835 bytes, 331983 tape blocks
1022026-03-17T06:13:51.344Zx out/crucible-pantry-zone.tar.gz, 43734698 bytes, 85420 tape blocks
1032026-03-17T06:13:51.344Zx out/crucible-utils.tar, 18169344 bytes, 35487 tape blocks
1042026-03-17T06:13:51.344Zx out/crucible-zone.tar.gz, 58212119 bytes, 113696 tape blocks
1052026-03-17T06:13:51.344Zx out/external-dns.tar.gz, 51742204 bytes, 101059 tape blocks
1062026-03-17T06:13:51.344Zx out/internal-dns.tar.gz, 51741800 bytes, 101059 tape blocks
1072026-03-17T06:13:51.344Zx out/mg-ddm-gz.tar, 80258048 bytes, 156754 tape blocks
1082026-03-17T06:13:51.344Zx out/nexus.tar.gz, 221900226 bytes, 433399 tape blocks
1092026-03-17T06:13:51.344Zx out/ntp.tar.gz, 31911439 bytes, 62328 tape blocks
1102026-03-17T06:13:51.344Zx out/omicron-gateway-softnpu.tar.gz, 27988426 bytes, 54665 tape blocks
1112026-03-17T06:13:51.344Zx out/omicron-sled-agent.tar, 205324800 bytes, 401025 tape blocks
1122026-03-17T06:13:51.344Zx out/overlay.tar.gz, 3030 bytes, 6 tape blocks
1132026-03-17T06:13:51.369Zx out/oximeter.tar.gz, 67399776 bytes, 131641 tape blocks
1142026-03-17T06:13:51.369Zx out/oxlog.tar, 3871744 bytes, 7562 tape blocks
1152026-03-17T06:13:51.369Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1162026-03-17T06:13:51.369Zx out/propolis-server.tar.gz, 76424187 bytes, 149266 tape blocks
1172026-03-17T06:13:51.369Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1182026-03-17T06:13:51.369Zx out/switch-softnpu.tar.gz, 388406909 bytes, 758608 tape blocks
1192026-03-17T06:13:51.369Z
1202026-03-17T06:13:51.369Zreal 21.557139620
1212026-03-17T06:13:51.370Zuser 19.298009646
1222026-03-17T06:13:51.370Zsys 3.837703155
1232026-03-17T06:13:51.370Ztrap 0.000918207
1242026-03-17T06:13:51.370Ztflt 0.000000000
1252026-03-17T06:13:51.370Zdflt 0.000000000
1262026-03-17T06:13:51.370Zkflt 0.000000000
1272026-03-17T06:13:51.370Zlock 0.000000000
1282026-03-17T06:13:51.370Zslp 19.437375748
1292026-03-17T06:13:51.370Zlat 0.538027579
1302026-03-17T06:13:51.370Zstop 0.000126461
1312026-03-17T06:13:51.370Z+ source .github/buildomat/ci-env.sh
1322026-03-17T06:13:51.370Z++ export CARGO_TERM_COLOR=always
1332026-03-17T06:13:51.370Z++ CARGO_TERM_COLOR=always
1342026-03-17T06:13:51.370Z+ bmat address ls
1352026-03-17T06:13:51.370ZNAME CIDR FIRST LAST COUNT
1362026-03-17T06:13:51.370Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1372026-03-17T06:13:51.370Z++ bmat address ls -f extra -Ho first,last,count
1382026-03-17T06:13:51.370Z+ set -- 10.151.2.164 10.151.2.227 64
1392026-03-17T06:13:51.370Z+ EXTRA_IP_START=10.151.2.164
1402026-03-17T06:13:51.370Z+ EXTRA_IP_END=10.151.2.227
1412026-03-17T06:13:51.370Z+ EXTRA_IP_COUNT=64
1422026-03-17T06:13:51.370Z+ (( EXTRA_IP_COUNT >= 32 ))
1432026-03-17T06:13:51.370Z+ EXTRA_IP_BASE=10.151.2
1442026-03-17T06:13:51.370Z+ EXTRA_IP_FOCTET=164
1452026-03-17T06:13:51.370Z+ EXTRA_IP_LOCTET=227
1462026-03-17T06:13:51.370Z+ SERVICE_IP_POOL_START=10.151.2.164
1472026-03-17T06:13:51.370Z+ SERVICE_IP_POOL_END=10.151.2.173
1482026-03-17T06:13:51.370Z+ DNS_IP1=10.151.2.164
1492026-03-17T06:13:51.370Z+ DNS_IP2=10.151.2.165
1502026-03-17T06:13:51.370Z+ UPLINK_IP=10.151.2.174
1512026-03-17T06:13:51.370Z+ IPPOOL_START=10.151.2.175
1522026-03-17T06:13:51.370Z+ IPPOOL_END=10.151.2.227
1532026-03-17T06:13:51.395Z++ ipadm show-addr -po type,addr
1542026-03-17T06:13:51.395Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1552026-03-17T06:13:51.396Z+ GATEWAY_IP=10.151.2.100
1562026-03-17T06:13:51.396Z+ [[ -n 10.151.2.100 ]]
1572026-03-17T06:13:51.396Z+ ping -s 10.151.2.100 56 1
1582026-03-17T06:13:51.396ZPING 10.151.2.100: 56 data bytes
1592026-03-17T06:13:51.396Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.011 ms
1602026-03-17T06:13:51.396Z
1612026-03-17T06:13:51.396Z----10.151.2.100 PING Statistics----
1622026-03-17T06:13:51.396Z1 packets transmitted, 1 packets received, 0% packet loss
1632026-03-17T06:13:51.396Zround-trip (ms) min/avg/max/stddev = 0.011/0.011/0.011/-nan
1642026-03-17T06:13:51.396Z++ arp -an
1652026-03-17T06:13:51.396Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1662026-03-17T06:13:51.396Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1672026-03-17T06:13:51.396Z+ routeadm -e ipv4-forwarding -u
1682026-03-17T06:13:51.495Z+ PXA_START=10.151.2.164
1692026-03-17T06:13:51.495Z+ PXA_END=10.151.2.227
1702026-03-17T06:13:51.495Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
1712026-03-17T06:13:51.627Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1722026-03-17T06:13:51.652ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1732026-03-17T06:13:51.699Zcreating virtual hardware
1742026-03-17T06:13:51.724Zcreating /scratch/m2_0.vdev
1752026-03-17T06:13:51.724Zcreating /scratch/m2_1.vdev
1762026-03-17T06:13:51.724Zcreating /scratch/u2_0.vdev
1772026-03-17T06:13:51.724Zcreating /scratch/u2_1.vdev
1782026-03-17T06:13:51.724Zcreating /scratch/u2_2.vdev
1792026-03-17T06:13:51.724Zcreating /scratch/u2_3.vdev
1802026-03-17T06:13:51.724Zcreating /scratch/u2_4.vdev
1812026-03-17T06:13:51.724Zcreating /scratch/u2_5.vdev
1822026-03-17T06:13:51.724Zcreating /scratch/u2_6.vdev
1832026-03-17T06:13:51.724Zcreating /scratch/u2_7.vdev
1842026-03-17T06:13:51.724Zcreating /scratch/u2_8.vdev
1852026-03-17T06:13:51.749ZSimnet net0/sc0_0 exists
1862026-03-17T06:13:51.773ZSimnet net1/sc1_0 exists
1872026-03-17T06:13:51.773ZVnic sc0_1 exists
1882026-03-17T06:14:04.806ZUsing 10.151.2.100 as gateway ip
1892026-03-17T06:14:04.806Zusing 18:C0:4D:0D:A0:2A as gateway mac
1902026-03-17T06:14:04.806Zconfiguring SoftNPU ARP entry
1912026-03-17T06:14:04.832Zconfiguring SoftNPU proxy ARP
1922026-03-17T06:14:04.884ZSoftNPU state:
1932026-03-17T06:14:04.884Z local v6:
1942026-03-17T06:14:04.884Z local v4:
1952026-03-17T06:14:04.884Z router v6_idx:
1962026-03-17T06:14:04.884Z router v6_routes:
1972026-03-17T06:14:04.884Z router v4_idx:
1982026-03-17T06:14:04.884Z router v4_routes:
1992026-03-17T06:14:04.884Z resolver v4:
2002026-03-17T06:14:04.884Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
2012026-03-17T06:14:04.884Z resolver v6:
2022026-03-17T06:14:04.884Z nat v4:
2032026-03-17T06:14:04.884Z nat_v6:
2042026-03-17T06:14:04.884Z attached subnet v4:
2052026-03-17T06:14:04.884Z attached subnet v6:
2062026-03-17T06:14:04.884Z port_mac:
2072026-03-17T06:14:04.884Z proxy arp:
2082026-03-17T06:14:04.885Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2092026-03-17T06:14:04.885Zcreated virtual hardware
2102026-03-17T06:14:04.909Z
2112026-03-17T06:14:04.909Zreal 13.255327012
2122026-03-17T06:14:04.909Zuser 0.657244166
2132026-03-17T06:14:04.909Zsys 3.360339871
2142026-03-17T06:14:04.909Ztrap 0.000079549
2152026-03-17T06:14:04.909Ztflt 0.000336988
2162026-03-17T06:14:04.909Zdflt 0.000545189
2172026-03-17T06:14:04.909Zkflt 0.000008480
2182026-03-17T06:14:04.909Zlock 10:15.887360716
2192026-03-17T06:14:04.909Zslp 40.039021512
2202026-03-17T06:14:04.909Zlat 0.133781769
2212026-03-17T06:14:04.909Zstop 0.237512254
2222026-03-17T06:14:04.909Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2232026-03-17T06:14:04.950Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2242026-03-17T06:14:04.997Z+ diff -u pkg/config.toml~ pkg/config.toml
2252026-03-17T06:14:04.998Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2262026-03-17T06:14:04.998Z+++ pkg/config.toml Tue Mar 17 06:14:05 2026
2272026-03-17T06:14:04.998Z@@ -29,18 +29,18 @@
2282026-03-17T06:14:04.998Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
2292026-03-17T06:14:04.998Z # followed by a string that is embedded into their fake serial values.
2302026-03-17T06:14:04.998Z vdevs = [
2312026-03-17T06:14:04.998Z- "m2_0.vdev",
2322026-03-17T06:14:04.998Z- "m2_1.vdev",
2332026-03-17T06:14:04.998Z+ "/scratch/m2_0.vdev",
2342026-03-17T06:14:04.998Z+ "/scratch/m2_1.vdev",
2352026-03-17T06:14:04.998Z
2362026-03-17T06:14:04.998Z- "u2_0.vdev",
2372026-03-17T06:14:04.998Z- "u2_1.vdev",
2382026-03-17T06:14:04.998Z- "u2_2.vdev",
2392026-03-17T06:14:04.998Z- "u2_3.vdev",
2402026-03-17T06:14:04.998Z- "u2_4.vdev",
2412026-03-17T06:14:04.998Z- "u2_5.vdev",
2422026-03-17T06:14:04.998Z- "u2_6.vdev",
2432026-03-17T06:14:04.998Z- "u2_7.vdev",
2442026-03-17T06:14:04.998Z- "u2_8.vdev",
2452026-03-17T06:14:04.998Z+ "/scratch/u2_0.vdev",
2462026-03-17T06:14:04.998Z+ "/scratch/u2_1.vdev",
2472026-03-17T06:14:04.998Z+ "/scratch/u2_2.vdev",
2482026-03-17T06:14:04.998Z+ "/scratch/u2_3.vdev",
2492026-03-17T06:14:04.998Z+ "/scratch/u2_4.vdev",
2502026-03-17T06:14:04.998Z+ "/scratch/u2_5.vdev",
2512026-03-17T06:14:04.998Z+ "/scratch/u2_6.vdev",
2522026-03-17T06:14:04.998Z+ "/scratch/u2_7.vdev",
2532026-03-17T06:14:04.998Z+ "/scratch/u2_8.vdev",
2542026-03-17T06:14:04.998Z ]
2552026-03-17T06:14:04.998Z
2562026-03-17T06:14:04.998Z # The amount of memory held back for services which exist between zero and one
2572026-03-17T06:14:04.998Z+ true
2582026-03-17T06:14:04.998Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2592026-03-17T06:14:04.998Z+ EXPECTED_ZPOOL_COUNT=9
2602026-03-17T06:14:04.998Z+ echo 'expected number of zpools is 9'
2612026-03-17T06:14:04.998Zexpected number of zpools is 9
2622026-03-17T06:14:04.998Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2632026-03-17T06:14:04.999Z+ SILO_NAME=recovery
2642026-03-17T06:14:04.999Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2652026-03-17T06:14:04.999Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2662026-03-17T06:14:04.999Z+ sed -i~ '
2672026-03-17T06:14:04.999Z /^external_dns_ips/c\
2682026-03-17T06:14:04.999Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2692026-03-17T06:14:04.999Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
2702026-03-17T06:14:04.999Z /^first/c\
2712026-03-17T06:14:04.999Zfirst = "10.151.2.164"
2722026-03-17T06:14:04.999Z /^last/c\
2732026-03-17T06:14:04.999Zlast = "10.151.2.173"
2742026-03-17T06:14:04.999Z }
2752026-03-17T06:14:04.999Z /^infra_ip_first/c\
2762026-03-17T06:14:04.999Zinfra_ip_first = "10.151.2.174"
2772026-03-17T06:14:04.999Z /^infra_ip_last/c\
2782026-03-17T06:14:04.999Zinfra_ip_last = "10.151.2.174"
2792026-03-17T06:14:04.999Z /^\[\[rack_network_config.ports/,/^$/ {
2802026-03-17T06:14:04.999Z /^routes/c\
2812026-03-17T06:14:04.999Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2822026-03-17T06:14:04.999Z /^addresses/c\
2832026-03-17T06:14:04.999Zaddresses = \[{address = "10.151.2.174/24"} \]
2842026-03-17T06:14:04.999Z }
2852026-03-17T06:14:04.999Z' pkg/config-rss.toml
2862026-03-17T06:14:04.999Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2872026-03-17T06:14:04.999Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2882026-03-17T06:14:04.999Z+++ pkg/config-rss.toml Tue Mar 17 06:14:05 2026
2892026-03-17T06:14:04.999Z@@ -33,7 +33,7 @@
2902026-03-17T06:14:04.999Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2912026-03-17T06:14:04.999Z # must be contained in one of the "internal services" IP Pool ranges listed
2922026-03-17T06:14:04.999Z # below.
2932026-03-17T06:14:04.999Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2942026-03-17T06:14:04.999Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2952026-03-17T06:14:04.999Z
2962026-03-17T06:14:04.999Z # Initial TLS certificates for the external API
2972026-03-17T06:14:04.999Z #
2982026-03-17T06:14:04.999Z@@ -69,8 +69,8 @@
2992026-03-17T06:14:04.999Z #
3002026-03-17T06:14:04.999Z # For more on this and what to put here, see docs/how-to-run.adoc.
3012026-03-17T06:14:04.999Z [[internal_services_ip_pool_ranges]]
3022026-03-17T06:14:05.000Z-first = "192.168.1.20"
3032026-03-17T06:14:05.000Z-last = "192.168.1.29"
3042026-03-17T06:14:05.000Z+first = "10.151.2.164"
3052026-03-17T06:14:05.000Z+last = "10.151.2.173"
3062026-03-17T06:14:05.000Z
3072026-03-17T06:14:05.000Z # TODO - this configuration is subject to change going forward. Ultimately these
3082026-03-17T06:14:05.000Z # parameters should be provided to the control plane via wicket, but we need to
3092026-03-17T06:14:05.000Z@@ -91,8 +91,8 @@
3102026-03-17T06:14:05.000Z # A range of IP addresses used by Boundary Services on the external network. In
3112026-03-17T06:14:05.000Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3122026-03-17T06:14:05.000Z # With softnpu, only one address is used.
3132026-03-17T06:14:05.000Z-infra_ip_first = "192.168.1.30"
3142026-03-17T06:14:05.000Z-infra_ip_last = "192.168.1.30"
3152026-03-17T06:14:05.000Z+infra_ip_first = "10.151.2.174"
3162026-03-17T06:14:05.000Z+infra_ip_last = "10.151.2.174"
3172026-03-17T06:14:05.000Z
3182026-03-17T06:14:05.000Z # Configurations for BGP routers to run on the scrimlets.
3192026-03-17T06:14:05.000Z bgp = []
3202026-03-17T06:14:05.000Z@@ -100,9 +100,9 @@
3212026-03-17T06:14:05.000Z # You can configure multiple uplinks by repeating the following stanza
3222026-03-17T06:14:05.000Z [[rack_network_config.ports]]
3232026-03-17T06:14:05.000Z # Routes associated with this port.
3242026-03-17T06:14:05.025Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3252026-03-17T06:14:05.026Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3262026-03-17T06:14:05.026Z # Addresses associated with this port.
3272026-03-17T06:14:05.026Z-addresses = [{address = "192.168.1.30/24"}]
3282026-03-17T06:14:05.026Z+addresses = [{address = "10.151.2.174/24"} ]
3292026-03-17T06:14:05.026Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3302026-03-17T06:14:05.026Z port = "qsfp0"
3312026-03-17T06:14:05.026Z # The speed of this port.
3322026-03-17T06:14:05.026Z+ true
3332026-03-17T06:14:05.026Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3342026-03-17T06:14:05.026Za pkg/config-rss.toml 8K
3352026-03-17T06:14:05.026Za pkg/config.toml 6K
3362026-03-17T06:14:05.026Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3372026-03-17T06:14:05.026Z+ TLS_NAME=recovery.sys.oxide.test
3382026-03-17T06:14:05.026Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3392026-03-17T06:14:05.652Z.......+...+.........+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+............+...+............+.......+.....+...+...+....+...+...+..+...+.........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+...+...............+..+.+........+......+...+..........+.....+......+.......+........+.......+..............+.........+.........+......+....+......+............+.....+....+......+.....+.......+..................+..+...............+..................+....+.........+......+......+...+..+...+.........+......+....+....................+..................+.............+..............................+......+........................+.....+....+..+...+.+......+...+........+....+...+........+.............+.....+.+.....................+.....+.......+...........+.........+.......+...+..+................+..+......+.+........+......+.......+..+.+..+.......+..+.+..................+..+.......+...+...+...............+......+...............+...+...+..+................+..+...+....+..+.+.........+...+....................+.....................+...............+.+.....+....+...........+...+.......+...+..+.+..+......+.+.....+.+.....+.......+..+....+..+...............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3402026-03-17T06:14:06.243Z.........+.+...........+.......+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..................+..+............+......+....+..+.+.....+.........+..........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...............+.........+......+...+.....+............+.+.........+......+..+.............+..+................+.........+...+.....+...............+.......+............+...+..+...+.............+............+..+.+........+....+..................+.........+.....+......+...+....+...........+....+...........+..........+.....+.+........+..........+..................+...+...........+....+..+....+..................+.................+.............+...+...............+........+......+.............+......+..+...+....+...+...+.....+..........+.....+.......+.....+....+........+..........+..+.+........+......+....+..+.........+....+.............................+.+............+..+.+............+...............+..................+....................+.+....................+...+............+...+.+.........+...........+......+.+.................+.+..............+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3412026-03-17T06:14:06.267Z-----
3422026-03-17T06:14:06.267Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3432026-03-17T06:14:06.292Za pkg/initial-tls-cert.pem 2K
3442026-03-17T06:14:06.292Za pkg/initial-tls-key.pem 4K
3452026-03-17T06:14:06.292Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3462026-03-17T06:14:06.292Z+ rmdir pkg
3472026-03-17T06:14:06.292Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3482026-03-17T06:14:06.292Z+ pfexec mkdir -p /usr/oxide
3492026-03-17T06:14:06.292Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3502026-03-17T06:14:06.956Z+ pfexec chmod +x /usr/oxide/oxide
3512026-03-17T06:14:06.981Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3522026-03-17T06:14:25.339Z+ OMICRON_NO_UNINSTALL=1
3532026-03-17T06:14:25.339Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3542026-03-17T06:14:25.364ZLogging to: /opt/oxide/work/out/LOG
3552026-03-17T06:14:26.204Z
3562026-03-17T06:14:26.204Zreal 0.859356514
3572026-03-17T06:14:26.204Zuser 0.173478973
3582026-03-17T06:14:26.229Zsys 5.326333157
3592026-03-17T06:14:26.229Ztrap 0.000002780
3602026-03-17T06:14:26.229Ztflt 0.000161160
3612026-03-17T06:14:26.229Zdflt 0.000200062
3622026-03-17T06:14:26.229Zkflt 0.000000000
3632026-03-17T06:14:26.229Zlock 1:17.282625342
3642026-03-17T06:14:26.229Zslp 1.040782331
3652026-03-17T06:14:26.229Zlat 0.029272703
3662026-03-17T06:14:26.229Zstop 0.063377507
3672026-03-17T06:14:26.229Z+ retry=0
3682026-03-17T06:14:26.229Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3692026-03-17T06:14:26.229Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3702026-03-17T06:14:26.229Z+ [[ 0 -gt 30 ]]
3712026-03-17T06:14:26.229Z+ sleep 1
3722026-03-17T06:14:27.218Z+ retry=1
3732026-03-17T06:14:27.218Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3742026-03-17T06:14:27.243Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3752026-03-17T06:14:27.243Z+ [[ 1 -gt 30 ]]
3762026-03-17T06:14:27.243Z+ sleep 1
 
3772026-03-17T06:14:28.122Zsock console login: Mar 17 06:14:28 sock svc.startd[8]: oxide/sled-agent:default failed repeatedly: transitioned to maintenance (see 'svcs -xv' for details)
3782026-03-17T06:14:28.175ZSUNW-MSG-ID: SMF-8000-YX, TYPE: defect, VER: 1, SEVERITY: major
 
3792026-03-17T06:14:28.232Z+ retry=2
3802026-03-17T06:14:28.232Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
 
3812026-03-17T06:14:28.252ZEVENT-TIME: Tue Mar 17 06:14:28 UTC 2026
 
3822026-03-17T06:14:28.256Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3832026-03-17T06:14:28.256Z+ [[ 2 -gt 30 ]]
3842026-03-17T06:14:28.256Z+ sleep 1
 
3852026-03-17T06:14:28.299ZPLATFORM: R152-Z32-00, CSN: GKG8N7212A0003, HOSTNAME: sock
3862026-03-17T06:14:28.334ZSOURCE: software-diagnosis, REV: 0.1
3872026-03-17T06:14:28.369ZEVENT-ID: f8b304b3-ecaf-4d75-ac38-67928e85e8ff
3882026-03-17T06:14:28.404ZDESC: A service failed - the instance is restarting too quickly.
3892026-03-17T06:14:28.438ZRefer to http://illumos.org/msg/SMF-8000-YX for more information.
3902026-03-17T06:14:28.473ZAUTO-RESPONSE: The service has been placed into the maintenance state.
3912026-03-17T06:14:28.508ZIMPACT: svc:/oxide/sled-agent:default is unavailable.
3922026-03-17T06:14:28.543ZREC-ACTION: Run 'svcs -xv svc:/oxide/sled-agent:default' to determine the generic reason why the service failed, the location of any logfiles, and a list of other services impacted.
 
3932026-03-17T06:14:29.245Z+ retry=3
3942026-03-17T06:14:29.246Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3952026-03-17T06:14:29.270Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3962026-03-17T06:14:29.270Z+ [[ 3 -gt 30 ]]
3972026-03-17T06:14:29.270Z+ sleep 1
3982026-03-17T06:14:30.259Z+ retry=4
3992026-03-17T06:14:30.259Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4002026-03-17T06:14:30.284Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4012026-03-17T06:14:30.284Z+ [[ 4 -gt 30 ]]
4022026-03-17T06:14:30.284Z+ sleep 1
4032026-03-17T06:14:31.273Z+ retry=5
4042026-03-17T06:14:31.273Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4052026-03-17T06:14:31.298Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4062026-03-17T06:14:31.298Z+ [[ 5 -gt 30 ]]
4072026-03-17T06:14:31.298Z+ sleep 1
4082026-03-17T06:14:32.286Z+ retry=6
4092026-03-17T06:14:32.286Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4102026-03-17T06:14:32.310Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4112026-03-17T06:14:32.311Z+ [[ 6 -gt 30 ]]
4122026-03-17T06:14:32.311Z+ sleep 1
4132026-03-17T06:14:33.301Z+ retry=7
4142026-03-17T06:14:33.301Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4152026-03-17T06:14:33.326Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4162026-03-17T06:14:33.326Z+ [[ 7 -gt 30 ]]
4172026-03-17T06:14:33.326Z+ sleep 1
4182026-03-17T06:14:34.314Z+ retry=8
4192026-03-17T06:14:34.314Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4202026-03-17T06:14:34.339Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4212026-03-17T06:14:34.339Z+ [[ 8 -gt 30 ]]
4222026-03-17T06:14:34.339Z+ sleep 1
4232026-03-17T06:14:35.328Z+ retry=9
4242026-03-17T06:14:35.328Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4252026-03-17T06:14:35.353Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4262026-03-17T06:14:35.353Z+ [[ 9 -gt 30 ]]
4272026-03-17T06:14:35.353Z+ sleep 1
4282026-03-17T06:14:36.341Z+ retry=10
4292026-03-17T06:14:36.341Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4302026-03-17T06:14:36.366Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4312026-03-17T06:14:36.366Z+ [[ 10 -gt 30 ]]
4322026-03-17T06:14:36.366Z+ sleep 1
4332026-03-17T06:14:37.355Z+ retry=11
4342026-03-17T06:14:37.355Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4352026-03-17T06:14:37.379Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4362026-03-17T06:14:37.379Z+ [[ 11 -gt 30 ]]
4372026-03-17T06:14:37.379Z+ sleep 1
4382026-03-17T06:14:38.369Z+ retry=12
4392026-03-17T06:14:38.369Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4402026-03-17T06:14:38.394Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4412026-03-17T06:14:38.394Z+ [[ 12 -gt 30 ]]
4422026-03-17T06:14:38.394Z+ sleep 1
4432026-03-17T06:14:39.382Z+ retry=13
4442026-03-17T06:14:39.382Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4452026-03-17T06:14:39.407Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4462026-03-17T06:14:39.407Z+ [[ 13 -gt 30 ]]
4472026-03-17T06:14:39.407Z+ sleep 1
4482026-03-17T06:14:40.396Z+ retry=14
4492026-03-17T06:14:40.396Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4502026-03-17T06:14:40.421Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4512026-03-17T06:14:40.421Z+ [[ 14 -gt 30 ]]
4522026-03-17T06:14:40.421Z+ sleep 1
4532026-03-17T06:14:41.410Z+ retry=15
4542026-03-17T06:14:41.410Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4552026-03-17T06:14:41.435Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4562026-03-17T06:14:41.435Z+ [[ 15 -gt 30 ]]
4572026-03-17T06:14:41.435Z+ sleep 1
4582026-03-17T06:14:42.422Z+ retry=16
4592026-03-17T06:14:42.422Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4602026-03-17T06:14:42.447Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4612026-03-17T06:14:42.447Z+ [[ 16 -gt 30 ]]
4622026-03-17T06:14:42.447Z+ sleep 1
4632026-03-17T06:14:43.436Z+ retry=17
4642026-03-17T06:14:43.436Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4652026-03-17T06:14:43.461Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4662026-03-17T06:14:43.461Z+ [[ 17 -gt 30 ]]
4672026-03-17T06:14:43.461Z+ sleep 1
4682026-03-17T06:14:44.449Z+ retry=18
4692026-03-17T06:14:44.449Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4702026-03-17T06:14:44.474Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4712026-03-17T06:14:44.474Z+ [[ 18 -gt 30 ]]
4722026-03-17T06:14:44.474Z+ sleep 1
4732026-03-17T06:14:45.462Z+ retry=19
4742026-03-17T06:14:45.462Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4752026-03-17T06:14:45.487Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4762026-03-17T06:14:45.487Z+ [[ 19 -gt 30 ]]
4772026-03-17T06:14:45.487Z+ sleep 1
4782026-03-17T06:14:46.475Z+ retry=20
4792026-03-17T06:14:46.475Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4802026-03-17T06:14:46.500Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4812026-03-17T06:14:46.500Z+ [[ 20 -gt 30 ]]
4822026-03-17T06:14:46.500Z+ sleep 1
4832026-03-17T06:14:47.489Z+ retry=21
4842026-03-17T06:14:47.490Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4852026-03-17T06:14:47.514Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4862026-03-17T06:14:47.514Z+ [[ 21 -gt 30 ]]
4872026-03-17T06:14:47.514Z+ sleep 1
4882026-03-17T06:14:48.505Z+ retry=22
4892026-03-17T06:14:48.505Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4902026-03-17T06:14:48.529Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4912026-03-17T06:14:48.529Z+ [[ 22 -gt 30 ]]
4922026-03-17T06:14:48.530Z+ sleep 1
4932026-03-17T06:14:49.518Z+ retry=23
4942026-03-17T06:14:49.518Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4952026-03-17T06:14:49.543Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
4962026-03-17T06:14:49.543Z+ [[ 23 -gt 30 ]]
4972026-03-17T06:14:49.543Z+ sleep 1
4982026-03-17T06:14:50.532Z+ retry=24
4992026-03-17T06:14:50.532Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5002026-03-17T06:14:50.557Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5012026-03-17T06:14:50.557Z+ [[ 24 -gt 30 ]]
5022026-03-17T06:14:50.557Z+ sleep 1
5032026-03-17T06:14:51.545Z+ retry=25
5042026-03-17T06:14:51.545Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5052026-03-17T06:14:51.569Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5062026-03-17T06:14:51.569Z+ [[ 25 -gt 30 ]]
5072026-03-17T06:14:51.569Z+ sleep 1
5082026-03-17T06:14:52.559Z+ retry=26
5092026-03-17T06:14:52.559Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5102026-03-17T06:14:52.584Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5112026-03-17T06:14:52.584Z+ [[ 26 -gt 30 ]]
5122026-03-17T06:14:52.584Z+ sleep 1
5132026-03-17T06:14:53.573Z+ retry=27
5142026-03-17T06:14:53.573Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5152026-03-17T06:14:53.598Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5162026-03-17T06:14:53.598Z+ [[ 27 -gt 30 ]]
5172026-03-17T06:14:53.598Z+ sleep 1
5182026-03-17T06:14:54.586Z+ retry=28
5192026-03-17T06:14:54.586Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5202026-03-17T06:14:54.611Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5212026-03-17T06:14:54.611Z+ [[ 28 -gt 30 ]]
5222026-03-17T06:14:54.611Z+ sleep 1
5232026-03-17T06:14:55.600Z+ retry=29
5242026-03-17T06:14:55.600Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5252026-03-17T06:14:55.625Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5262026-03-17T06:14:55.625Z+ [[ 29 -gt 30 ]]
5272026-03-17T06:14:55.625Z+ sleep 1
5282026-03-17T06:14:56.612Z+ retry=30
5292026-03-17T06:14:56.613Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5302026-03-17T06:14:56.637Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5312026-03-17T06:14:56.637Z+ [[ 30 -gt 30 ]]
5322026-03-17T06:14:56.637Z+ sleep 1
5332026-03-17T06:14:57.626Z+ retry=31
5342026-03-17T06:14:57.626Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5352026-03-17T06:14:57.651Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
5362026-03-17T06:14:57.651Z+ [[ 31 -gt 30 ]]
5372026-03-17T06:14:57.651Z+ echo 'Failed to reach switch zone after 30 attempts'
5382026-03-17T06:14:57.651Z+ exit 1
5392026-03-17T06:14:57.651ZFailed to reach switch zone after 30 attempts
5402026-03-17T06:14:57.651Z+ _exit_trap
5412026-03-17T06:14:57.651Z+ local status=1
5422026-03-17T06:14:57.651Z+ set +o errexit
5432026-03-17T06:14:57.651Z+ [[ x != \x ]]
5442026-03-17T06:14:57.651Z+ pfexec svcadm -Z disable -s cron
5452026-03-17T06:14:57.690Z+ [[ 1 -eq 0 ]]
5462026-03-17T06:14:57.690Z+ set -o xtrace
5472026-03-17T06:14:57.690Z+ banner evidence
5482026-03-17T06:14:57.715Z
5492026-03-17T06:14:57.715Z ###### # # # ##### ###### # # #### ######
5502026-03-17T06:14:57.715Z # # # # # # # ## # # # #
5512026-03-17T06:14:57.715Z ##### # # # # # ##### # # # # #####
5522026-03-17T06:14:57.715Z # # # # # # # # # # # #
5532026-03-17T06:14:57.715Z # # # # # # # # ## # # #
5542026-03-17T06:14:57.715Z ###### ## # ##### ###### # # #### ######
5552026-03-17T06:14:57.715Z
5562026-03-17T06:14:57.715Z+ zoneadm list -civ
5572026-03-17T06:14:57.715Z ID NAME STATUS PATH BRAND IP
5582026-03-17T06:14:57.715Z 0 global running / ipkg shared
5592026-03-17T06:14:57.715Z 1 sidecar_softnpu running /sidecar/sidecar_softnpu omicron1 excl
5602026-03-17T06:14:57.715Z+ pfexec dladm show-phys -m
5612026-03-17T06:14:57.715ZLINK SLOT ADDRESS INUSE CLIENT
5622026-03-17T06:14:57.716Zigb0 primary 18:c0:4d:d:a0:2a yes igb0
5632026-03-17T06:14:57.716Zigb1 primary 18:c0:4d:d:a0:2b no --
5642026-03-17T06:14:57.716Z+ pfexec dladm show-link
5652026-03-17T06:14:57.716ZLINK CLASS MTU STATE BRIDGE OVER
5662026-03-17T06:14:57.716Zigb0 phys 1500 up -- --
5672026-03-17T06:14:57.716Zigb1 phys 1500 unknown -- --
5682026-03-17T06:14:57.716Znet0 simnet 9000 up -- sc0_0
5692026-03-17T06:14:57.716Zsc0_0 simnet 9000 up -- net0
5702026-03-17T06:14:57.716Znet1 simnet 9000 up -- sc1_0
5712026-03-17T06:14:57.716Zsc1_0 simnet 9000 up -- net1
5722026-03-17T06:14:57.716Zsc0_1 vnic 1500 up -- igb0
5732026-03-17T06:14:57.716Ztfportrear0_0 simnet 9000 up -- sidecar_shi0
5742026-03-17T06:14:57.716Zsidecar_shi0 simnet 9000 up -- tfportrear0_0
5752026-03-17T06:14:57.716Ztfportqsfp0_0 simnet 9000 up -- sidecar_shi1
5762026-03-17T06:14:57.716Zsidecar_shi1 simnet 9000 up -- tfportqsfp0_0
5772026-03-17T06:14:57.716Zbootstrap_stub0 etherstub 9000 up -- --
5782026-03-17T06:14:57.716Zbootstrap0 vnic 1500 up -- bootstrap_stub0
5792026-03-17T06:14:57.716Zunderlay_stub0 etherstub 9000 up -- --
5802026-03-17T06:14:57.716Zunderlay0 vnic 9000 up -- underlay_stub0
5812026-03-17T06:14:57.741Z+ pfexec dladm show-vnic
5822026-03-17T06:14:57.741ZLINK OVER SPEED MACADDRESS MACADDRTYPE VID
5832026-03-17T06:14:57.741Zsc0_1 igb0 1000 a8:e1:de:1:70:1d fixed 0
5842026-03-17T06:14:57.741Zbootstrap0 bootstrap_stub0 0 2:8:20:99:ce:ea random 0
5852026-03-17T06:14:57.741Zunderlay0 underlay_stub0 0 2:8:20:f8:59:fc random 0
5862026-03-17T06:14:57.741Z+ pfexec ipadm
5872026-03-17T06:14:57.741ZADDROBJ TYPE STATE ADDR
5882026-03-17T06:14:57.741Zlo0/v4 static ok 127.0.0.1/8
5892026-03-17T06:14:57.741Zigb0/dhcp dhcp ok 10.151.2.100/24
5902026-03-17T06:14:57.741Zlo0/v6 static ok ::1/128
5912026-03-17T06:14:57.741Znet0/ll addrconf ok fe80::50b8:d0ff:fe64:3e40%net0/10
5922026-03-17T06:14:57.741Znet1/ll addrconf ok fe80::c015:84ff:fedc:8985%net1/10
5932026-03-17T06:14:57.741Zbootstrap0/ll addrconf ok fe80::8:20ff:fe99:ceea%bootstrap0/10
5942026-03-17T06:14:57.741Zbootstrap0/bootstrap6 static ok fdb0:18c0:4d0d:a02a::1/64
5952026-03-17T06:14:57.741Z+ pfexec netstat -rncva
5962026-03-17T06:14:57.741Z
5972026-03-17T06:14:57.741ZIRE Table: IPv4
5982026-03-17T06:14:57.741Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
5992026-03-17T06:14:57.741Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
6002026-03-17T06:14:57.741Z0.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6012026-03-17T06:14:57.741Zdefault 0.0.0.0 10.151.2.1 igb0 1500 9 UG 1697679 0
6022026-03-17T06:14:57.741Z10.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6032026-03-17T06:14:57.741Z10.151.2.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6042026-03-17T06:14:57.742Z10.151.2.0/24 255.255.255.0 10.151.2.100 igb0 1500 3 U 0 0
6052026-03-17T06:14:57.742Z10.151.2.1/32 255.255.255.255 10.151.2.100 igb0 1500 3 UC 0 0
6062026-03-17T06:14:57.742Z10.151.2.100 255.255.255.255 -- igb0 1500 2 UHL 2 3390962
6072026-03-17T06:14:57.742Z10.151.2.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6082026-03-17T06:14:57.742Z10.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6092026-03-17T06:14:57.742Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 56 56
6102026-03-17T06:14:57.742Z255.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 6
6112026-03-17T06:14:57.742Z
6122026-03-17T06:14:57.742ZIRE Table: IPv6
6132026-03-17T06:14:57.742Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
6142026-03-17T06:14:57.742Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
6152026-03-17T06:14:57.742Z::1 ::1 lo0 8252 2 UH 28 28
6162026-03-17T06:14:57.742Zfdb0:18c0:4d0d:a02a::1 -- bootstrap0 1500 2 UHL 0 0
6172026-03-17T06:14:57.742Zfe80::8:20ff:fe99:ceea -- bootstrap0 1500 2 UHL 0 0
6182026-03-17T06:14:57.742Zfe80::50b8:d0ff:fe64:3e40 -- net0 9000 2 UHL 0 0
6192026-03-17T06:14:57.742Zfe80::c015:84ff:fedc:8985 -- net1 9000 2 UHL 0 0
6202026-03-17T06:14:57.742Zfdb0:18c0:4d0d:a02a::/64 fdb0:18c0:4d0d:a02a::1 bootstrap0 1500 2 U 0 0
6212026-03-17T06:14:57.742Zfe80::/10 fe80::8:20ff:fe99:ceea bootstrap0 1500 2 U 0 0
6222026-03-17T06:14:57.742Zfe80::/10 fe80::c015:84ff:fedc:8985 net1 9000 2 U 0 0
6232026-03-17T06:14:57.742Zfe80::/10 fe80::50b8:d0ff:fe64:3e40 net0 9000 2 U 0 0
6242026-03-17T06:14:57.742Z+ pfexec netstat -anu
6252026-03-17T06:14:57.767Z
6262026-03-17T06:14:57.767ZUDP: IPv4
6272026-03-17T06:14:57.767Z Local Address Remote Address User Pid Command State
6282026-03-17T06:14:57.767Z-------------------- -------------------- -------- ------ -------------- ----------
6292026-03-17T06:14:57.767Z *.* root 71 in.mpathd Unbound
6302026-03-17T06:14:57.767Z *.* root 71 in.mpathd Unbound
6312026-03-17T06:14:57.767Z *.68 root 304 dhcpagent Idle
6322026-03-17T06:14:57.767Z *.546 root 304 dhcpagent Idle
6332026-03-17T06:14:57.767Z10.151.2.100.68 root 304 dhcpagent Idle
6342026-03-17T06:14:57.767Z *.111 daemon 710 rpcbind Idle
6352026-03-17T06:14:57.767Z *.* daemon 710 rpcbind Unbound
6362026-03-17T06:14:57.768Z *.45846 daemon 710 rpcbind Idle
6372026-03-17T06:14:57.768Z *.111 daemon 710 rpcbind Idle
6382026-03-17T06:14:57.768Z *.* daemon 710 rpcbind Unbound
6392026-03-17T06:14:57.768Z *.58836 daemon 710 rpcbind Idle
6402026-03-17T06:14:57.768Z *.* root 721 in.ndpd Unbound
6412026-03-17T06:14:57.768Z *.* root 868 rcm_daemon Unbound
6422026-03-17T06:14:57.768Z *.* root 868 rcm_daemon Unbound
6432026-03-17T06:14:57.768Z
6442026-03-17T06:14:57.768ZUDP: IPv6
6452026-03-17T06:14:57.768Z Local Address Remote Address User Pid Command State If
6462026-03-17T06:14:57.768Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
6472026-03-17T06:14:57.768Z *.* root 71 in.mpathd Unbound
6482026-03-17T06:14:57.768Z *.546 root 304 dhcpagent Idle
6492026-03-17T06:14:57.768Z *.111 daemon 710 rpcbind Idle
6502026-03-17T06:14:57.768Z *.* daemon 710 rpcbind Unbound
6512026-03-17T06:14:57.768Z *.45846 daemon 710 rpcbind Idle
6522026-03-17T06:14:57.768Z *.* root 721 in.ndpd Unbound
6532026-03-17T06:14:57.768Z *.* root 868 rcm_daemon Unbound
6542026-03-17T06:14:57.768Zff02::dd.3549 root 1575 ddmd Idle net1
6552026-03-17T06:14:57.768Zfe80::c015:84ff:fedc:8985.3549 root 1575 ddmd Idle net1
6562026-03-17T06:14:57.768Zff02::dd.3549 root 1575 ddmd Idle net0
6572026-03-17T06:14:57.768Zfe80::50b8:d0ff:fe64:3e40.3549 root 1575 ddmd Idle net0
6582026-03-17T06:14:57.768Z
6592026-03-17T06:14:57.768ZTCP: IPv4
6602026-03-17T06:14:57.768Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State
6612026-03-17T06:14:57.768Z-------------------- -------------------- -------- ------ -------------- ------ ------ ------ ------ -----------
6622026-03-17T06:14:57.768Z127.0.0.1.5999 *.* root 71 in.mpathd 0 0 128000 0 LISTEN
6632026-03-17T06:14:57.768Z127.0.0.1.4999 *.* root 304 dhcpagent 0 0 128000 0 LISTEN
6642026-03-17T06:14:57.768Z *.111 *.* daemon 710 rpcbind 0 0 128000 0 LISTEN
6652026-03-17T06:14:57.768Z *.* *.* daemon 710 rpcbind 0 0 128000 0 IDLE
6662026-03-17T06:14:57.768Z *.111 *.* daemon 710 rpcbind 0 0 128000 0 LISTEN
6672026-03-17T06:14:57.768Z *.* *.* daemon 710 rpcbind 0 0 128000 0 IDLE
6682026-03-17T06:14:57.768Z10.151.2.100.48915 54.187.123.254.443 root 802 agent 1049792 0 128872 0 ESTABLISHED
6692026-03-17T06:14:57.768Z10.151.2.100.57389 54.187.123.254.443 root 802 agent 1049792 0 128872 0 ESTABLISHED
6702026-03-17T06:14:57.768Z10.151.2.100.35295 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6712026-03-17T06:14:57.768Z10.151.2.100.35438 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6722026-03-17T06:14:57.768Z10.151.2.100.52617 54.187.123.254.443 1049792 0 128872 0 TIME_WAIT
6732026-03-17T06:14:57.768Z10.151.2.100.37009 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
6742026-03-17T06:14:57.769Z10.151.2.100.46892 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
6752026-03-17T06:14:57.769Z
6762026-03-17T06:14:57.769ZTCP: IPv6
6772026-03-17T06:14:57.769Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State If
6782026-03-17T06:14:57.769Z--------------------------------- --------------------------------- -------- ------ -------------- ------ ------ ------ ------ ----------- -----
6792026-03-17T06:14:57.769Z::1.5999 *.* root 71 in.mpathd 0 0 128000 0 LISTEN
6802026-03-17T06:14:57.769Z *.111 *.* daemon 710 rpcbind 0 0 128000 0 LISTEN
6812026-03-17T06:14:57.769Z *.* *.* daemon 710 rpcbind 0 0 128000 0 IDLE
6822026-03-17T06:14:57.769Z::1.8000 *.* root 1575 ddmd 0 0 128000 0 LISTEN
6832026-03-17T06:14:57.769Z
6842026-03-17T06:14:57.769ZActive UNIX domain sockets
6852026-03-17T06:14:57.769ZAddress Type User Pid Command Local Address Remote Address
6862026-03-17T06:14:57.769Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
6872026-03-17T06:14:57.769Zfffffe85b71d4b48 stream-ord root 1575 ddmd
6882026-03-17T06:14:57.769Zfffffe85b71d4b48 stream-ord root 1575 ddmd
6892026-03-17T06:14:57.769Zfffffe85b7131060 stream-ord root 1575 ddmd
6902026-03-17T06:14:57.769Zfffffe85a6aafb18 stream-ord root 888 zoneadmd /var/run/zones/sidecar_softnpu.console_
6912026-03-17T06:14:57.769Zfffffe8586f9f758 stream-ord root 802 agent
6922026-03-17T06:14:57.769Zfffffe8586f9f758 stream-ord root 802 agent
6932026-03-17T06:14:57.769Zfffffe8586f9faf8 stream-ord root 802 agent
6942026-03-17T06:14:57.769Zfffffe8586e42010 stream-ord root 802 agent /var/run/buildomat.sock
6952026-03-17T06:14:57.769Zfffffe85844a6008 stream-ord root 757 inetd /var/run/.inetd.uds
6962026-03-17T06:14:57.769Zfffffe85844a63a8 dgram root 721 in.ndpd /var/run/in.ndpd_mib
6972026-03-17T06:14:57.769Zfffffe85844a6748 stream-ord root 721 in.ndpd /var/run/in.ndpd_ipadm
6982026-03-17T06:14:57.769Z+ pfexec arp -an
6992026-03-17T06:14:57.769ZNet to Media Table: IPv4
7002026-03-17T06:14:57.769ZDevice IP Address Mask Flags Phys Addr
7012026-03-17T06:14:57.769Z------ -------------------- --------------- -------- ---------------
7022026-03-17T06:14:57.769Zigb0 10.151.2.1 255.255.255.255 aa:00:04:00:ca:fe
7032026-03-17T06:14:57.769Zigb0 10.151.2.100 255.255.255.255 SPLA 18:c0:4d:0d:a0:2a
7042026-03-17T06:14:57.769Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
7052026-03-17T06:14:57.794Zlocal v6:
7062026-03-17T06:14:57.794Zlocal v4:
7072026-03-17T06:14:57.794Zrouter v6_idx:
7082026-03-17T06:14:57.794Zrouter v6_routes:
7092026-03-17T06:14:57.794Zrouter v4_idx:
7102026-03-17T06:14:57.794Zrouter v4_routes:
7112026-03-17T06:14:57.794Zresolver v4:
7122026-03-17T06:14:57.794Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
7132026-03-17T06:14:57.795Zresolver v6:
7142026-03-17T06:14:57.795Znat v4:
7152026-03-17T06:14:57.795Znat_v6:
7162026-03-17T06:14:57.795Zattached subnet v4:
7172026-03-17T06:14:57.795Zattached subnet v6:
7182026-03-17T06:14:57.795Zport_mac:
7192026-03-17T06:14:57.795Zproxy arp:
7202026-03-17T06:14:57.795Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
7212026-03-17T06:14:57.795Z+ pfexec /opt/oxide/opte/bin/opteadm list-ports
7222026-03-17T06:14:57.795ZLINK MAC ADDRESS IPv4 ADDRESS EPHEMERAL IPv4 FLOATING IPv4 IPv6 ADDRESS EXTERNAL IPv6 FLOATING IPv6 STATE
7232026-03-17T06:14:57.795Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2b
7242026-03-17T06:14:57.795ZVirtual to Boundary Mappings
7252026-03-17T06:14:57.795Z======================================================================
7262026-03-17T06:14:57.795Z
7272026-03-17T06:14:57.795ZIPv4 mappings
7282026-03-17T06:14:57.795Z----------------------------------------------------------------------
7292026-03-17T06:14:57.795ZTUNNELED PREFIX BOUNDARY IP VNI
7302026-03-17T06:14:57.795Z
7312026-03-17T06:14:57.795ZIPv6 mappings
7322026-03-17T06:14:57.795Z----------------------------------------------------------------------
7332026-03-17T06:14:57.795ZTUNNELED PREFIX BOUNDARY IP VNI
7342026-03-17T06:14:57.795Z
7352026-03-17T06:14:57.795Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2p
7362026-03-17T06:14:57.795ZVirtual to Physical Mappings
7372026-03-17T06:14:57.795Z======================================================================
7382026-03-17T06:14:57.795Z+ z_swadm link ls
7392026-03-17T06:14:57.795Z+ echo '== swadm link' ls
7402026-03-17T06:14:57.795Z== swadm link ls
7412026-03-17T06:14:57.795Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm link ls
7422026-03-17T06:14:57.795Zzlogin: zone 'oxz_switch' unknown
7432026-03-17T06:14:57.795Z+ z_swadm addr list
7442026-03-17T06:14:57.795Z+ echo '== swadm addr' list
7452026-03-17T06:14:57.795Z== swadm addr list
7462026-03-17T06:14:57.795Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm addr list
7472026-03-17T06:14:57.820Zzlogin: zone 'oxz_switch' unknown
7482026-03-17T06:14:57.820Z+ z_swadm route list
7492026-03-17T06:14:57.820Z+ echo '== swadm route' list
7502026-03-17T06:14:57.820Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm route list
7512026-03-17T06:14:57.820Z== swadm route list
7522026-03-17T06:14:57.820Zzlogin: zone 'oxz_switch' unknown
7532026-03-17T06:14:57.820Z+ z_swadm arp list
7542026-03-17T06:14:57.820Z+ echo '== swadm arp' list
7552026-03-17T06:14:57.820Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm arp list
7562026-03-17T06:14:57.820Z== swadm arp list
7572026-03-17T06:14:57.820Zzlogin: zone 'oxz_switch' unknown
7582026-03-17T06:14:57.820Z+ z_swadm nat list
7592026-03-17T06:14:57.821Z+ echo '== swadm nat' list
7602026-03-17T06:14:57.821Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm nat list
7612026-03-17T06:14:57.821Z== swadm nat list
7622026-03-17T06:14:57.821Zzlogin: zone 'oxz_switch' unknown
7632026-03-17T06:14:57.821Z++ pfexec /opt/oxide/opte/bin/opteadm list-ports
7642026-03-17T06:14:57.821Z++ tail +2
7652026-03-17T06:14:57.821Z++ awk '{ print $1; }'
7662026-03-17T06:14:57.821Z+ PORTS=
7672026-03-17T06:14:57.821Z+ mkdir -p /tmp/kstat
7682026-03-17T06:14:57.821Z+ pfexec kstat -p xde:
7692026-03-17T06:14:57.845Z+ pfexec zfs list
7702026-03-17T06:14:57.845ZNAME USED AVAIL REFER MOUNTPOINT
7712026-03-17T06:14:57.846Zrpool 2.34G 4.92G 24K /rpool
7722026-03-17T06:14:57.846Zrpool/home 25.5K 4.92G 25.5K /home
7732026-03-17T06:14:57.846Zrpool/input 2.27G 4.92G 2.27G /input
7742026-03-17T06:14:57.846Zrpool/sidecar 75.8M 4.92G 24K /sidecar
7752026-03-17T06:14:57.846Zrpool/sidecar/sidecar_softnpu 75.7M 4.92G 75.7M /sidecar/sidecar_softnpu
7762026-03-17T06:14:57.846Zrpool/work 24K 4.92G 24K /work
7772026-03-17T06:14:57.846Zrpool/zone 24K 4.92G 24K /zone
7782026-03-17T06:14:57.846Zscratch 115K 1.80T 25K /scratch
7792026-03-17T06:14:57.846Z+ pfexec zpool list
7802026-03-17T06:14:57.846ZNAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
7812026-03-17T06:14:57.846Zrpool 7.50G 2.34G 5.16G - - 0% 31% 1.00x ONLINE -
7822026-03-17T06:14:57.846Zscratch 1.86T 115K 1.86T - - 0% 0% 1.00x ONLINE -
7832026-03-17T06:14:57.846Z+ pfexec fmdump -eVp
7842026-03-17T06:14:57.846ZTIME CLASS
7852026-03-17T06:14:57.846Z+ pfexec ptree -z global
7862026-03-17T06:14:57.870Z8 /lib/svc/bin/svc.startd
7872026-03-17T06:14:57.870Z 639 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
7882026-03-17T06:14:57.871Z 655 /usr/lib/saf/sac -t 300
7892026-03-17T06:14:57.871Z 665 /usr/lib/saf/ttymon
7902026-03-17T06:14:57.871Z 801 /usr/bin/ctrun -l child -o noorphan,regent /opt/buildomat/lib/agent ru
7912026-03-17T06:14:57.871Z 802 /opt/buildomat/lib/agent run
7922026-03-17T06:14:57.871Z 816 /bin/bash /tmp/yXVgMQiIaMEFb1qQ.sh
7932026-03-17T06:14:57.871Z 1782 ptree -z global
7942026-03-17T06:14:57.871Z10 /lib/svc/bin/svc.configd
7952026-03-17T06:14:57.871Z39 /lib/inet/ipmgmtd
7962026-03-17T06:14:57.871Z40 /lib/inet/netcfgd
7972026-03-17T06:14:57.871Z44 /sbin/dlmgmtd
7982026-03-17T06:14:57.871Z71 /lib/inet/in.mpathd
7992026-03-17T06:14:57.871Z91 /usr/lib/pfexecd
8002026-03-17T06:14:57.871Z175 /usr/lib/zones/zonestatd
8012026-03-17T06:14:57.871Z187 /usr/lib/sysevent/syseventd
8022026-03-17T06:14:57.871Z198 devfsadmd
8032026-03-17T06:14:57.871Z304 /sbin/dhcpagent
8042026-03-17T06:14:57.871Z565 /usr/lib/picl/picld
8052026-03-17T06:14:57.871Z608 /usr/sbin/nscd
8062026-03-17T06:14:57.871Z638 /usr/lib/utmpd
8072026-03-17T06:14:57.871Z660 /usr/lib/hotplugd
8082026-03-17T06:14:57.871Z673 /usr/sbin/syslogd
8092026-03-17T06:14:57.871Z710 /usr/sbin/rpcbind
8102026-03-17T06:14:57.871Z721 /usr/lib/inet/in.ndpd
8112026-03-17T06:14:57.871Z722 /usr/lib/fm/fmd/fmd
8122026-03-17T06:14:57.871Z757 /usr/lib/inet/inetd start
8132026-03-17T06:14:57.871Z782 /usr/perl5/bin/perl /usr/lib/intrd
8142026-03-17T06:14:57.871Z868 /usr/lib/rcm/rcm_daemon
8152026-03-17T06:14:57.871Z888 zoneadmd -z sidecar_softnpu
8162026-03-17T06:14:57.871Z1573 ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_scrip
8172026-03-17T06:14:57.871Z 1575 /opt/oxide/mg-ddm/ddmd --with-stats --admin-port 8000 --admin-addr ::1
8182026-03-17T06:14:57.871Z+ pfexec svcs -xv
8192026-03-17T06:14:57.918Zsvc:/oxide/sled-agent:default (Oxide Sled Agent)
8202026-03-17T06:14:57.918Z State: maintenance since March 17, 2026 at 06:14:28 AM UTC
8212026-03-17T06:14:57.918ZReason: Restarting too quickly.
8222026-03-17T06:14:57.918Z See: http://illumos.org/msg/SMF-8000-L5
8232026-03-17T06:14:57.918Z See: /var/svc/log/oxide-sled-agent:default.log
8242026-03-17T06:14:57.918ZImpact: This service is not running.
8252026-03-17T06:14:57.944Z++ zoneadm list -n
8262026-03-17T06:14:57.944Z+ for z in $(zoneadm list -n)
8272026-03-17T06:14:57.944Z+ banner sidecar_softnpu
8282026-03-17T06:14:57.944Z
8292026-03-17T06:14:57.944Z #### # ##### ###### #### ## ##### #### ####
8302026-03-17T06:14:57.944Z # # # # # # # # # # # # # #
8312026-03-17T06:14:57.944Z #### # # # ##### # # # # # #### # #
8322026-03-17T06:14:57.944Z # # # # # # ###### ##### # # #
8332026-03-17T06:14:57.944Z # # # # # # # # # # # # # # # #
8342026-03-17T06:14:57.944Z #### # ##### ###### #### # # # # ####### #### ####
8352026-03-17T06:14:57.944Z
8362026-03-17T06:14:57.944Z+ pfexec svcs -xv -z sidecar_softnpu
8372026-03-17T06:14:57.985Z+ pfexec ptree -z sidecar_softnpu
8382026-03-17T06:14:58.010Z891 zsched
8392026-03-17T06:14:58.010Z 908 /sbin/init
8402026-03-17T06:14:58.010Z913 /lib/svc/bin/svc.startd
8412026-03-17T06:14:58.010Z 1458 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
8422026-03-17T06:14:58.010Z915 /lib/svc/bin/svc.configd
8432026-03-17T06:14:58.010Z959 /lib/inet/ipmgmtd
8442026-03-17T06:14:58.010Z960 /lib/inet/netcfgd
8452026-03-17T06:14:58.010Z1154 /usr/lib/fm/fmd/fmd
8462026-03-17T06:14:58.010Z1173 /usr/lib/pfexecd
8472026-03-17T06:14:58.010Z1289 /softnpu/softnpu --uds-path /softnpu /softnpu/softnpu.toml
8482026-03-17T06:14:58.010Z1391 /usr/sbin/nscd
8492026-03-17T06:14:58.010Z1441 /usr/lib/utmpd
8502026-03-17T06:14:58.010Z1456 /usr/sbin/syslogd
8512026-03-17T06:14:58.010Z1514 /usr/lib/inet/in.ndpd
8522026-03-17T06:14:58.010Z+ pfexec zlogin sidecar_softnpu ipadm
8532026-03-17T06:14:58.035ZADDROBJ TYPE STATE ADDR
8542026-03-17T06:14:58.035Zlo0/v4 static ok 127.0.0.1/8
8552026-03-17T06:14:58.035Zlo0/v6 static ok ::1/128
8562026-03-17T06:14:58.035Z+ pfexec zlogin sidecar_softnpu netstat -rncva
8572026-03-17T06:14:58.035Z
8582026-03-17T06:14:58.035ZIRE Table: IPv4
8592026-03-17T06:14:58.035Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
8602026-03-17T06:14:58.035Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
8612026-03-17T06:14:58.035Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 0 0
8622026-03-17T06:14:58.035Z
8632026-03-17T06:14:58.035ZIRE Table: IPv6
8642026-03-17T06:14:58.035Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
8652026-03-17T06:14:58.035Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
8662026-03-17T06:14:58.035Z::1 ::1 lo0 8252 2 UH 0 0
8672026-03-17T06:14:58.035Z+ pfexec zlogin sidecar_softnpu netstat -anu
8682026-03-17T06:14:58.060Z
8692026-03-17T06:14:58.060ZUDP: IPv4
8702026-03-17T06:14:58.060Z Local Address Remote Address User Pid Command State
8712026-03-17T06:14:58.060Z-------------------- -------------------- -------- ------ -------------- ----------
8722026-03-17T06:14:58.060Z *.* root 1514 in.ndpd Unbound
8732026-03-17T06:14:58.060Z
8742026-03-17T06:14:58.060ZUDP: IPv6
8752026-03-17T06:14:58.060Z Local Address Remote Address User Pid Command State If
8762026-03-17T06:14:58.060Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
8772026-03-17T06:14:58.060Z *.* root 1514 in.ndpd Unbound
8782026-03-17T06:14:58.060Z
8792026-03-17T06:14:58.060ZActive UNIX domain sockets
8802026-03-17T06:14:58.060ZAddress Type User Pid Command Local Address Remote Address
8812026-03-17T06:14:58.060Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
8822026-03-17T06:14:58.060Zfffffe85b2d5fb30 stream-ord root 1514 in.ndpd /var/run/in.ndpd_ipadm
8832026-03-17T06:14:58.060Zfffffe85a6aaf038 stream-ord root 1289 softnpu
8842026-03-17T06:14:58.060Zfffffe85a6aaf038 stream-ord root 1289 softnpu
8852026-03-17T06:14:58.060Zfffffe85a6aaf3d8 stream-ord root 1289 softnpu
8862026-03-17T06:14:58.060Zfffffe85a6aaf778 dgram root 1289 softnpu /softnpu/server
8872026-03-17T06:14:58.060Zfffffe8586e42af0 dgram root 1514 in.ndpd /var/run/in.ndpd_mib
8882026-03-17T06:14:58.060Z+ pfexec zlogin sidecar_softnpu arp -an
8892026-03-17T06:14:58.085Z++ zoneadm list -n
8902026-03-17T06:14:58.085Z++ grep oxz_ntp
8912026-03-17T06:14:58.085Z+ pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log
8922026-03-17T06:14:58.110ZConfig {
8932026-03-17T06:14:58.110Z p4_program: "/softnpu/asic_program.so",
8942026-03-17T06:14:58.110Z ports: [
8952026-03-17T06:14:58.110Z Port {
8962026-03-17T06:14:58.110Z sidecar: "sc0_0",
8972026-03-17T06:14:58.110Z scrimlet: "sidecar_shi0",
8982026-03-17T06:14:58.110Z mtu: 9000,
8992026-03-17T06:14:58.110Z },
9002026-03-17T06:14:58.110Z Port {
9012026-03-17T06:14:58.110Z sidecar: "sc0_1",
9022026-03-17T06:14:58.110Z scrimlet: "sidecar_shi1",
9032026-03-17T06:14:58.110Z mtu: 9000,
9042026-03-17T06:14:58.110Z },
9052026-03-17T06:14:58.110Z ],
9062026-03-17T06:14:58.111Z}
9072026-03-17T06:14:58.111Z+ exit 1
9082026-03-17T06:14:58.111Zprocess exited: duration 88535 ms, exit code 1
 
9092026-03-17T06:14:58.161Zfound 3 output files
9102026-03-17T06:14:58.187Zuploading: /var/svc/log/oxide-mg-ddm:default.log (4405 bytes)
9112026-03-17T06:14:59.260Zuploaded: /var/svc/log/oxide-mg-ddm:default.log
9122026-03-17T06:14:59.260Zuploading: /var/svc/log/oxide-sled-agent:default.log (13348 bytes)
9132026-03-17T06:15:00.378Zuploaded: /var/svc/log/oxide-sled-agent:default.log
9142026-03-17T06:15:00.378Zuploading: /tmp/kstat/xde.kstat (890 bytes)
9152026-03-17T06:15:01.474Zuploaded: /tmp/kstat/xde.kstat