01HZ6KNYYE2MT9MECPDQ9A006J: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-sled-agent:default.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-2.0-opte-0.28",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HZ6KPFK8YPYBZR68VNHVF936

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-05-31T06:38:30.544Zjob dependencies complete; ready to run (waiting for 39 m 7 s)
22024-05-31T06:42:27.437Zjob assigned to worker 01HZ6P08TJKVWA1HMYNAW345E9 (queued for 3 m 56 s)
32024-05-31T06:42:27.551Zhttp://172.20.0.5:9969/ipxe/buskin... ok
42024-05-31T06:42:27.717Zhttp://172.20.0.5:9969/os/buskin/platform/i86pc/kernel/amd64/unix... ok
52024-05-31T06:42:27.771Zhttp://172.20.0.5:9969/os/buskin/platform/i86pc/amd64/boot_archive... ok
62024-05-31T06:42:27.805Zhttp://172.20.0.5:9969/os/buskin/platform/i86pc/amd64/boot_archive.hash... ok
72024-05-31T06:42:27.838Zhttp://172.20.0.5:9969/postboot/buskin... ok
82024-05-31T06:42:27.872ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92024-05-31T06:42:27.908Zcpu0: microcode has been updated from version 0xa001133 to 0xa0011d3
102024-05-31T06:42:27.938ZOxide Helios Version helios-2.0.22656 64-bit
112024-05-31T06:42:27.972ZConfiguring devices.
122024-05-31T06:42:28.005Z##################################################
132024-05-31T06:42:28.051Z#### ############# ##
142024-05-31T06:42:28.095Z### ### ############ ##
152024-05-31T06:42:28.128Z## ### # ## ### ## Oxide Computer Company ##
162024-05-31T06:42:28.162Z## ## ## ### # ### ##
172024-05-31T06:42:28.196Z## # ### #### #### This Station Under ##
182024-05-31T06:42:28.228Z### ### #### # ### Computer Control ##
192024-05-31T06:42:28.262Z#### #### ### ## ##
202024-05-31T06:42:28.295Z##################################################
212024-05-31T06:42:28.328ZHostname: buskin
222024-05-31T06:42:28.365ZLoading smf(7) service descriptions:
232024-05-31T06:42:28.726Zdownloading input: /input/ci-tools/work/end-to-end-tests/bootstrap-513fc87449fc9212.gz
242024-05-31T06:42:30.309Zdownloaded input: /input/ci-tools/work/end-to-end-tests/bootstrap-513fc87449fc9212.gz
252024-05-31T06:42:30.335Zdownloading input: /input/ci-tools/work/end-to-end-tests/bootstrap.gz
262024-05-31T06:42:54.532Zdownloaded input: /input/ci-tools/work/end-to-end-tests/bootstrap.gz
272024-05-31T06:42:54.558Zdownloading input: /input/ci-tools/work/end-to-end-tests/commtest-9cfbc31e704d0c76.gz
282024-05-31T06:42:56.013Zdownloaded input: /input/ci-tools/work/end-to-end-tests/commtest-9cfbc31e704d0c76.gz
292024-05-31T06:42:56.013Zdownloading input: /input/ci-tools/work/end-to-end-tests/dhcp_server-14dd6501e12a054f.gz
302024-05-31T06:42:57.500Zdownloaded input: /input/ci-tools/work/end-to-end-tests/dhcp_server-14dd6501e12a054f.gz
312024-05-31T06:42:57.500Zdownloading input: /input/ci-tools/work/end-to-end-tests/end_to_end_tests-54e84b08c5696f77.gz
322024-05-31T06:43:17.781Zdownloaded input: /input/ci-tools/work/end-to-end-tests/end_to_end_tests-54e84b08c5696f77.gz
332024-05-31T06:43:17.781Zdownloading input: /input/ci-tools/work/caboose-util.gz
342024-05-31T06:43:18.844Zdownloaded input: /input/ci-tools/work/caboose-util.gz
352024-05-31T06:43:18.869Zdownloading input: /input/ci-tools/work/tufaceous.gz
362024-05-31T06:43:21.339Zdownloaded input: /input/ci-tools/work/tufaceous.gz
372024-05-31T06:43:21.364Zdownloading input: /input/ci-tools/work/commtest
382024-05-31T06:44:45.534Zdownloaded input: /input/ci-tools/work/commtest
392024-05-31T06:44:45.558Zdownloading input: /input/ci-tools/work/permslip.gz
402024-05-31T06:44:47.552Zdownloaded input: /input/ci-tools/work/permslip.gz
412024-05-31T06:44:47.577Zdownloading input: /input/package/work/version.txt
422024-05-31T06:44:47.602Zdownloaded input: /input/package/work/version.txt
432024-05-31T06:44:47.602Zdownloading input: /input/package/work/package.tar.gz
442024-05-31T06:45:02.409Zdownloaded input: /input/package/work/package.tar.gz
452024-05-31T06:45:02.437Zdownloading input: /input/package/work/global-zone-packages.tar.gz
462024-05-31T06:45:29.096Zdownloaded input: /input/package/work/global-zone-packages.tar.gz
472024-05-31T06:45:29.096Zdownloading input: /input/package/work/trampoline-global-zone-packages.tar.gz
482024-05-31T06:45:36.122Zdownloaded input: /input/package/work/trampoline-global-zone-packages.tar.gz
492024-05-31T06:45:36.148Zdownloading input: /input/package/work/zones/clickhouse.tar.gz
502024-05-31T06:46:22.471Zdownloaded input: /input/package/work/zones/clickhouse.tar.gz
512024-05-31T06:46:22.471Zdownloading input: /input/package/work/zones/clickhouse_keeper.tar.gz
522024-05-31T06:47:04.886Zdownloaded input: /input/package/work/zones/clickhouse_keeper.tar.gz
532024-05-31T06:47:04.911Zdownloading input: /input/package/work/zones/cockroachdb.tar.gz
542024-05-31T06:47:26.693Zdownloaded input: /input/package/work/zones/cockroachdb.tar.gz
552024-05-31T06:47:26.693Zdownloading input: /input/package/work/zones/crucible-pantry-zone.tar.gz
562024-05-31T06:47:31.744Zdownloaded input: /input/package/work/zones/crucible-pantry-zone.tar.gz
572024-05-31T06:47:31.744Zdownloading input: /input/package/work/zones/crucible-zone.tar.gz
582024-05-31T06:47:38.665Zdownloaded input: /input/package/work/zones/crucible-zone.tar.gz
592024-05-31T06:47:38.665Zdownloading input: /input/package/work/zones/external-dns.tar.gz
602024-05-31T06:47:43.264Zdownloaded input: /input/package/work/zones/external-dns.tar.gz
612024-05-31T06:47:43.289Zdownloading input: /input/package/work/zones/internal-dns.tar.gz
622024-05-31T06:47:47.941Zdownloaded input: /input/package/work/zones/internal-dns.tar.gz
632024-05-31T06:47:47.941Zdownloading input: /input/package/work/zones/nexus-single-sled.tar.gz
642024-05-31T06:48:02.889Zdownloaded input: /input/package/work/zones/nexus-single-sled.tar.gz
652024-05-31T06:48:02.889Zdownloading input: /input/package/work/zones/nexus.tar.gz
662024-05-31T06:48:18.035Zdownloaded input: /input/package/work/zones/nexus.tar.gz
672024-05-31T06:48:18.035Zdownloading input: /input/package/work/zones/ntp.tar.gz
682024-05-31T06:48:19.731Zdownloaded input: /input/package/work/zones/ntp.tar.gz
692024-05-31T06:48:19.731Zdownloading input: /input/package/work/zones/omicron-gateway-asic.tar.gz
702024-05-31T06:48:22.420Zdownloaded input: /input/package/work/zones/omicron-gateway-asic.tar.gz
712024-05-31T06:48:22.420Zdownloading input: /input/package/work/zones/omicron-gateway-softnpu.tar.gz
722024-05-31T06:48:25.112Zdownloaded input: /input/package/work/zones/omicron-gateway-softnpu.tar.gz
732024-05-31T06:48:25.137Zdownloading input: /input/package/work/zones/overlay.tar.gz
742024-05-31T06:48:25.165Zdownloaded input: /input/package/work/zones/overlay.tar.gz
752024-05-31T06:48:25.165Zdownloading input: /input/package/work/zones/oximeter.tar.gz
762024-05-31T06:48:32.636Zdownloaded input: /input/package/work/zones/oximeter.tar.gz
772024-05-31T06:48:32.636Zdownloading input: /input/package/work/zones/probe.tar.gz
782024-05-31T06:48:33.238Zdownloaded input: /input/package/work/zones/probe.tar.gz
792024-05-31T06:48:33.238Zdownloading input: /input/package/work/zones/propolis-server.tar.gz
802024-05-31T06:48:44.812Zdownloaded input: /input/package/work/zones/propolis-server.tar.gz
812024-05-31T06:48:44.812Zdownloading input: /input/package/work/zones/switch-asic.tar.gz
822024-05-31T06:49:36.102Zdownloaded input: /input/package/work/zones/switch-asic.tar.gz
832024-05-31T06:49:36.103Zdownloading input: /input/package/work/zones/switch-softnpu.tar.gz
842024-05-31T06:50:11.074Zdownloaded input: /input/package/work/zones/switch-softnpu.tar.gz
 
852024-05-31T06:50:11.074Zstarting task 0: "setup"
862024-05-31T06:50:11.099Z++ uname -s
872024-05-31T06:50:11.099Z+ kern=SunOS
882024-05-31T06:50:11.099Z+ case "$kern" in
892024-05-31T06:50:11.099Z+ groupadd -g 12345 build
902024-05-31T06:50:11.099Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
912024-05-31T06:50:13.096Z+ zfs create -o mountpoint=/work rpool/work
922024-05-31T06:50:13.121Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
932024-05-31T06:50:13.121Z+ home_fs=zfs
942024-05-31T06:50:13.121Z+ [[ zfs == autofs ]]
952024-05-31T06:50:13.121Z+ mkdir -p /home/build
962024-05-31T06:50:13.145Z+ chown build:build /home/build /work
972024-05-31T06:50:15.124Z+ chmod 0700 /home/build /work
982024-05-31T06:50:15.149Zprocess exited: duration 4054 ms, exit code 0
 
992024-05-31T06:50:15.197Zstarting task 1: "authentication"
1002024-05-31T06:50:15.243Zprocess exited: duration 45 ms, exit code 0
 
1012024-05-31T06:50:15.292Zstarting task 2: "build"
1022024-05-31T06:50:15.317Z+ trap _exit_trap EXIT
1032024-05-31T06:50:15.317Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
1042024-05-31T06:50:15.413Zsecpolicy_fs_linkdir: 0x55 = 0x55
1052024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
1062024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
1072024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+3: 0 = 0xe5
1082024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+4: 0 = 0xb8
1092024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+5: 0 = 0x1
1102024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
1112024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
1122024-05-31T06:50:15.413Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
1132024-05-31T06:50:15.438Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
1142024-05-31T06:50:15.438Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
1152024-05-31T06:50:15.438Z+ [[ -d /opt/oxide ]]
1162024-05-31T06:50:15.438Z+ pfexec mv /opt/oxide /opt/oxide-underneath
1172024-05-31T06:50:15.438Z+ pfexec mkdir /opt/oxide
1182024-05-31T06:50:15.438Z+ pfexec mount -F tmpfs -O swap /opt/oxide
1192024-05-31T06:50:15.438Z+ [[ -d /opt/oxide-underneath ]]
1202024-05-31T06:50:15.438Z+ cd /opt/oxide-underneath
1212024-05-31T06:50:15.439Z+ pfexec tar ceEp@/f - .
1222024-05-31T06:50:15.439Z+ cd /opt/oxide
1232024-05-31T06:50:15.439Z+ pfexec tar xveEp@/f -
1242024-05-31T06:50:15.524Zx ., 0 bytes, 0 tape blocks
1252024-05-31T06:50:15.524Zx ./opte, 0 bytes, 0 tape blocks
1262024-05-31T06:50:15.524Zx ./opte/bin, 0 bytes, 0 tape blocks
1272024-05-31T06:50:15.524Zx ./opte/bin/opteadm, 43425608 bytes, 84816 tape blocks
1282024-05-31T06:50:15.549Z+ pfexec rm -rf /opt/oxide-underneath
1292024-05-31T06:50:15.549Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
1302024-05-31T06:50:15.573Z+ pfexec mkdir /opt/oxide/work
1312024-05-31T06:50:15.573Z+ pfexec chown build:build /opt/oxide/work
1322024-05-31T06:50:15.573Z+ cd /opt/oxide/work
1332024-05-31T06:50:15.573Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
1342024-05-31T06:50:17.143ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
1352024-05-31T06:50:17.143Zx out/mg-ddm-gz.tar, 62732800 bytes, 122525 tape blocks
1362024-05-31T06:50:17.143Zx out/omicron-sled-agent.tar, 96884224 bytes, 189227 tape blocks
1372024-05-31T06:50:17.143Zx out/oxlog.tar, 2982400 bytes, 5825 tape blocks
1382024-05-31T06:50:17.143Zx out/pumpkind-gz.tar, 27304960 bytes, 53330 tape blocks
1392024-05-31T06:50:17.143Zx out/target/test, 75 bytes, 1 tape blocks
1402024-05-31T06:50:17.143Zx out/npuzone/npuzone, 23699288 bytes, 46288 tape blocks
1412024-05-31T06:50:17.143Zx package-manifest.toml, 27341 bytes, 54 tape blocks
1422024-05-31T06:50:17.143Zx smf/sled-agent/non-gimlet/config.toml, 3154 bytes, 7 tape blocks
1432024-05-31T06:50:17.143Zx target/release/omicron-package, 32253672 bytes, 62996 tape blocks
1442024-05-31T06:50:17.143Zx target/release/xtask, 6831760 bytes, 13344 tape blocks
1452024-05-31T06:50:17.168Z
1462024-05-31T06:50:17.168Zreal 1.579493838
1472024-05-31T06:50:17.168Zuser 1.427892668
1482024-05-31T06:50:17.168Zsys 0.269548371
1492024-05-31T06:50:17.169Ztrap 0.000064511
1502024-05-31T06:50:17.169Ztflt 0.000000000
1512024-05-31T06:50:17.169Zdflt 0.000000000
1522024-05-31T06:50:17.169Zkflt 0.000000000
1532024-05-31T06:50:17.169Zlock 0.000000000
1542024-05-31T06:50:17.169Zslp 1.425344478
1552024-05-31T06:50:17.169Zlat 0.034116410
1562024-05-31T06:50:17.169Zstop 0.000125360
1572024-05-31T06:50:17.169Z+ cp /input/package/work/zones/clickhouse_keeper.tar.gz /input/package/work/zones/clickhouse.tar.gz /input/package/work/zones/cockroachdb.tar.gz /input/package/work/zones/crucible-pantry-zone.tar.gz /input/package/work/zones/crucible-zone.tar.gz /input/package/work/zones/external-dns.tar.gz /input/package/work/zones/internal-dns.tar.gz /input/package/work/zones/nexus-single-sled.tar.gz /input/package/work/zones/nexus.tar.gz /input/package/work/zones/ntp.tar.gz /input/package/work/zones/omicron-gateway-asic.tar.gz /input/package/work/zones/omicron-gateway-softnpu.tar.gz /input/package/work/zones/overlay.tar.gz /input/package/work/zones/oximeter.tar.gz /input/package/work/zones/probe.tar.gz /input/package/work/zones/propolis-server.tar.gz /input/package/work/zones/switch-asic.tar.gz /input/package/work/zones/switch-softnpu.tar.gz out/
1582024-05-31T06:50:18.403Z+ mv out/nexus-single-sled.tar.gz out/nexus.tar.gz
1592024-05-31T06:50:18.427Z+ mkdir tests
1602024-05-31T06:50:18.427Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1612024-05-31T06:50:18.427Z+ ptime -m gunzip
1622024-05-31T06:50:18.428Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap-513fc87449fc9212
1632024-05-31T06:50:18.576Z
1642024-05-31T06:50:18.601Zreal 0.155275332
1652024-05-31T06:50:18.601Zuser 0.142793391
1662024-05-31T06:50:18.601Zsys 0.012248407
1672024-05-31T06:50:18.601Ztrap 0.000009589
1682024-05-31T06:50:18.601Ztflt 0.000000000
1692024-05-31T06:50:18.601Zdflt 0.000000000
1702024-05-31T06:50:18.601Zkflt 0.000000000
1712024-05-31T06:50:18.601Zlock 0.000000000
1722024-05-31T06:50:18.601Zslp 0.000019540
1732024-05-31T06:50:18.601Zlat 0.000016180
1742024-05-31T06:50:18.602Zstop 0.000119971
1752024-05-31T06:50:18.602Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap-513fc87449fc9212
1762024-05-31T06:50:18.602Z+ chmod a+x tests/bootstrap-513fc87449fc9212
1772024-05-31T06:50:18.602Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1782024-05-31T06:50:18.602Z+ ptime -m gunzip
1792024-05-31T06:50:18.602Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap
1802024-05-31T06:50:20.838Z
1812024-05-31T06:50:20.862Zreal 2.249743909
1822024-05-31T06:50:20.862Zuser 2.124181604
1832024-05-31T06:50:20.862Zsys 0.125159941
1842024-05-31T06:50:20.862Ztrap 0.000161580
1852024-05-31T06:50:20.862Ztflt 0.000000000
1862024-05-31T06:50:20.862Zdflt 0.000000000
1872024-05-31T06:50:20.862Zkflt 0.000000000
1882024-05-31T06:50:20.862Zlock 0.000000000
1892024-05-31T06:50:20.862Zslp 0.000022860
1902024-05-31T06:50:20.863Zlat 0.000030130
1912024-05-31T06:50:20.863Zstop 0.000123941
1922024-05-31T06:50:20.863Z++ basename /input/ci-tools/work/end-to-end-tests/bootstrap
1932024-05-31T06:50:20.863Z+ chmod a+x tests/bootstrap
1942024-05-31T06:50:20.863Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
1952024-05-31T06:50:20.863Z+ ptime -m gunzip
1962024-05-31T06:50:20.863Z++ basename /input/ci-tools/work/end-to-end-tests/commtest-9cfbc31e704d0c76
1972024-05-31T06:50:21.005Z
1982024-05-31T06:50:21.030Zreal 0.155145410
1992024-05-31T06:50:21.030Zuser 0.142753979
2002024-05-31T06:50:21.030Zsys 0.012171356
2012024-05-31T06:50:21.030Ztrap 0.000010769
2022024-05-31T06:50:21.030Ztflt 0.000000000
2032024-05-31T06:50:21.030Zdflt 0.000000000
2042024-05-31T06:50:21.030Zkflt 0.000000000
2052024-05-31T06:50:21.030Zlock 0.000000000
2062024-05-31T06:50:21.030Zslp 0.000016860
2072024-05-31T06:50:21.030Zlat 0.000016630
2082024-05-31T06:50:21.030Zstop 0.000116262
2092024-05-31T06:50:21.030Z++ basename /input/ci-tools/work/end-to-end-tests/commtest-9cfbc31e704d0c76
2102024-05-31T06:50:21.030Z+ chmod a+x tests/commtest-9cfbc31e704d0c76
2112024-05-31T06:50:21.031Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
2122024-05-31T06:50:21.031Z+ ptime -m gunzip
2132024-05-31T06:50:21.031Z++ basename /input/ci-tools/work/end-to-end-tests/dhcp_server-14dd6501e12a054f
2142024-05-31T06:50:21.169Z
2152024-05-31T06:50:21.194Zreal 0.153671825
2162024-05-31T06:50:21.194Zuser 0.142271380
2172024-05-31T06:50:21.194Zsys 0.011154870
2182024-05-31T06:50:21.194Ztrap 0.000008509
2192024-05-31T06:50:21.194Ztflt 0.000000000
2202024-05-31T06:50:21.194Zdflt 0.000000000
2212024-05-31T06:50:21.194Zkflt 0.000000000
2222024-05-31T06:50:21.194Zlock 0.000000000
2232024-05-31T06:50:21.194Zslp 0.000021850
2242024-05-31T06:50:21.195Zlat 0.000023040
2252024-05-31T06:50:21.195Zstop 0.000128842
2262024-05-31T06:50:21.195Z++ basename /input/ci-tools/work/end-to-end-tests/dhcp_server-14dd6501e12a054f
2272024-05-31T06:50:21.195Z+ chmod a+x tests/dhcp_server-14dd6501e12a054f
2282024-05-31T06:50:21.195Z+ for p in /input/ci-tools/work/end-to-end-tests/*.gz
2292024-05-31T06:50:21.195Z+ ptime -m gunzip
2302024-05-31T06:50:21.195Z++ basename /input/ci-tools/work/end-to-end-tests/end_to_end_tests-54e84b08c5696f77
2312024-05-31T06:50:23.482Z
2322024-05-31T06:50:23.506Zreal 2.300598111
2332024-05-31T06:50:23.506Zuser 2.157633734
2342024-05-31T06:50:23.506Zsys 0.142564899
2352024-05-31T06:50:23.506Ztrap 0.000159393
2362024-05-31T06:50:23.506Ztflt 0.000000000
2372024-05-31T06:50:23.507Zdflt 0.000000000
2382024-05-31T06:50:23.507Zkflt 0.000000000
2392024-05-31T06:50:23.507Zlock 0.000000000
2402024-05-31T06:50:23.507Zslp 0.000015520
2412024-05-31T06:50:23.507Zlat 0.000018850
2422024-05-31T06:50:23.507Zstop 0.000110072
2432024-05-31T06:50:23.507Z++ basename /input/ci-tools/work/end-to-end-tests/end_to_end_tests-54e84b08c5696f77
2442024-05-31T06:50:23.507Z+ chmod a+x tests/end_to_end_tests-54e84b08c5696f77
2452024-05-31T06:50:23.507Z+ bmat address ls
2462024-05-31T06:50:23.507ZNAME CIDR FIRST LAST COUNT
2472024-05-31T06:50:23.507Zextra 10.151.1.0/24 10.151.1.164 10.151.1.227 64
2482024-05-31T06:50:23.507Z++ bmat address ls -f extra -Ho first,last,count
2492024-05-31T06:50:23.532Z+ set -- 10.151.1.164 10.151.1.227 64
2502024-05-31T06:50:23.532Z+ EXTRA_IP_START=10.151.1.164
2512024-05-31T06:50:23.532Z+ EXTRA_IP_END=10.151.1.227
2522024-05-31T06:50:23.532Z+ EXTRA_IP_COUNT=64
2532024-05-31T06:50:23.532Z+ (( EXTRA_IP_COUNT >= 32 ))
2542024-05-31T06:50:23.532Z+ EXTRA_IP_BASE=10.151.1
2552024-05-31T06:50:23.532Z+ EXTRA_IP_FOCTET=164
2562024-05-31T06:50:23.532Z+ EXTRA_IP_LOCTET=227
2572024-05-31T06:50:23.532Z+ SERVICE_IP_POOL_START=10.151.1.164
2582024-05-31T06:50:23.532Z+ SERVICE_IP_POOL_END=10.151.1.173
2592024-05-31T06:50:23.532Z+ DNS_IP1=10.151.1.164
2602024-05-31T06:50:23.532Z+ DNS_IP2=10.151.1.165
2612024-05-31T06:50:23.532Z+ UPLINK_IP=10.151.1.174
2622024-05-31T06:50:23.532Z+ IPPOOL_START=10.151.1.175
2632024-05-31T06:50:23.532Z+ IPPOOL_END=10.151.1.227
2642024-05-31T06:50:23.532Z++ ipadm show-addr -po type,addr
2652024-05-31T06:50:23.532Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
2662024-05-31T06:50:23.532Z+ GATEWAY_IP=10.151.1.100
2672024-05-31T06:50:23.532Z+ [[ -n 10.151.1.100 ]]
2682024-05-31T06:50:23.532Z+ ping -s 10.151.1.100 56 1
2692024-05-31T06:50:23.532ZPING 10.151.1.100: 56 data bytes
2702024-05-31T06:50:23.533Z64 bytes from 10.151.1.100: icmp_seq=0. time=0.012 ms
2712024-05-31T06:50:23.533Z
2722024-05-31T06:50:23.533Z----10.151.1.100 PING Statistics----
2732024-05-31T06:50:23.533Z1 packets transmitted, 1 packets received, 0% packet loss
2742024-05-31T06:50:23.533Zround-trip (ms) min/avg/max/stddev = 0.012/0.012/0.012/-nan
2752024-05-31T06:50:23.533Z++ arp -an
2762024-05-31T06:50:23.533Z++ awk -vgw=10.151.1.100 '$2 == gw {print $NF}'
2772024-05-31T06:50:23.557Z+ GATEWAY_MAC=18:c0:4d:0d:9f:b2
2782024-05-31T06:50:23.557Z+ routeadm -e ipv4-forwarding -u
2792024-05-31T06:50:23.649Z+ PXA_START=10.151.1.164
2802024-05-31T06:50:23.649Z+ PXA_END=10.151.1.227
2812024-05-31T06:50:23.649Z+ pfexec zpool create -f scratch c1t1d0 c2t1d0
2822024-05-31T06:50:23.808Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.1.100 --gateway-mac 18:c0:4d:0d:9f:b2 --pxa-start 10.151.1.164 --pxa-end 10.151.1.227
2832024-05-31T06:50:23.836ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
2842024-05-31T06:50:23.885Zcreating virtual hardware
2852024-05-31T06:50:23.909Zcreating /scratch/m2_0.vdev
2862024-05-31T06:50:23.909Zcreating /scratch/m2_1.vdev
2872024-05-31T06:50:23.909Zcreating /scratch/u2_0.vdev
2882024-05-31T06:50:23.909Zcreating /scratch/u2_1.vdev
2892024-05-31T06:50:23.909Zcreating /scratch/u2_2.vdev
2902024-05-31T06:50:23.910Zcreating /scratch/u2_3.vdev
2912024-05-31T06:50:23.910Zcreating /scratch/u2_4.vdev
2922024-05-31T06:50:23.910Zcreating /scratch/u2_5.vdev
2932024-05-31T06:50:23.910Zcreating /scratch/u2_6.vdev
2942024-05-31T06:50:23.910Zcreating /scratch/u2_7.vdev
2952024-05-31T06:50:23.910Zcreating /scratch/u2_8.vdev
2962024-05-31T06:50:23.938ZSimnet net0/sc0_0 exists
2972024-05-31T06:50:23.965ZSimnet net1/sc1_0 exists
2982024-05-31T06:50:23.989ZVnic sc0_1 exists
2992024-05-31T06:50:40.399ZUsing 10.151.1.100 as gateway ip
3002024-05-31T06:50:40.399Zusing 18:C0:4D:0D:9F:B2 as gateway mac
3012024-05-31T06:50:40.399Zconfiguring SoftNPU ARP entry
3022024-05-31T06:50:40.427Zconfiguring SoftNPU proxy ARP
3032024-05-31T06:50:40.483ZSoftNPU state:
3042024-05-31T06:50:40.483Z local v6:
3052024-05-31T06:50:40.483Z local v4:
3062024-05-31T06:50:40.483Z router v6:
3072024-05-31T06:50:40.483Z router v4_idx:
3082024-05-31T06:50:40.483Z router v4_routes:
3092024-05-31T06:50:40.483Z resolver v4:
3102024-05-31T06:50:40.483Z 10.151.1.100 -> 18:c0:4d:0d:9f:b2
3112024-05-31T06:50:40.483Z resolver v6:
3122024-05-31T06:50:40.483Z nat v4:
3132024-05-31T06:50:40.483Z nat_v6:
3142024-05-31T06:50:40.483Z port_mac:
3152024-05-31T06:50:40.483Z proxy arp:
3162024-05-31T06:50:40.483Z 10.151.1.164/10.151.1.227: a8:e1:de:01:70:1d
3172024-05-31T06:50:40.484Zcreated virtual hardware
3182024-05-31T06:50:40.508Z
3192024-05-31T06:50:40.508Zreal 16.671462644
3202024-05-31T06:50:40.508Zuser 0.633545567
3212024-05-31T06:50:40.508Zsys 3.333448947
3222024-05-31T06:50:40.508Ztrap 0.000074257
3232024-05-31T06:50:40.508Ztflt 0.000100758
3242024-05-31T06:50:40.508Zdflt 0.001371002
3252024-05-31T06:50:40.508Zkflt 0.000003130
3262024-05-31T06:50:40.508Zlock 12:55.056948481
3272024-05-31T06:50:40.508Zslp 50.373763002
3282024-05-31T06:50:40.508Zlat 0.075909861
3292024-05-31T06:50:40.509Zstop 0.364028941
3302024-05-31T06:50:40.509Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3312024-05-31T06:50:40.556Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
3322024-05-31T06:50:40.556Z+ diff -u pkg/config.toml~ pkg/config.toml
3332024-05-31T06:50:40.556Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
3342024-05-31T06:50:40.556Z+++ pkg/config.toml Fri May 31 06:50:40 2024
3352024-05-31T06:50:40.556Z@@ -29,18 +29,18 @@
3362024-05-31T06:50:40.556Z # These paths have the prefix of either "u2" or "m2", followed by an underscore,
3372024-05-31T06:50:40.556Z # followed by a string that is embedded into their fake serial values.
3382024-05-31T06:50:40.556Z vdevs = [
3392024-05-31T06:50:40.557Z- "m2_0.vdev",
3402024-05-31T06:50:40.557Z- "m2_1.vdev",
3412024-05-31T06:50:40.557Z+ "/scratch/m2_0.vdev",
3422024-05-31T06:50:40.557Z+ "/scratch/m2_1.vdev",
3432024-05-31T06:50:40.557Z
3442024-05-31T06:50:40.557Z- "u2_0.vdev",
3452024-05-31T06:50:40.557Z- "u2_1.vdev",
3462024-05-31T06:50:40.557Z- "u2_2.vdev",
3472024-05-31T06:50:40.557Z- "u2_3.vdev",
3482024-05-31T06:50:40.557Z- "u2_4.vdev",
3492024-05-31T06:50:40.557Z- "u2_5.vdev",
3502024-05-31T06:50:40.557Z- "u2_6.vdev",
3512024-05-31T06:50:40.557Z- "u2_7.vdev",
3522024-05-31T06:50:40.557Z- "u2_8.vdev",
3532024-05-31T06:50:40.557Z+ "/scratch/u2_0.vdev",
3542024-05-31T06:50:40.557Z+ "/scratch/u2_1.vdev",
3552024-05-31T06:50:40.557Z+ "/scratch/u2_2.vdev",
3562024-05-31T06:50:40.557Z+ "/scratch/u2_3.vdev",
3572024-05-31T06:50:40.557Z+ "/scratch/u2_4.vdev",
3582024-05-31T06:50:40.557Z+ "/scratch/u2_5.vdev",
3592024-05-31T06:50:40.557Z+ "/scratch/u2_6.vdev",
3602024-05-31T06:50:40.557Z+ "/scratch/u2_7.vdev",
3612024-05-31T06:50:40.557Z+ "/scratch/u2_8.vdev",
3622024-05-31T06:50:40.557Z ]
3632024-05-31T06:50:40.558Z
3642024-05-31T06:50:40.558Z # Percentage of usable physical DRAM to use for the VMM reservoir, which
3652024-05-31T06:50:40.558Z+ true
3662024-05-31T06:50:40.558Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
3672024-05-31T06:50:40.558Z+ SILO_NAME=recovery
3682024-05-31T06:50:40.558Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
3692024-05-31T06:50:40.558Z+ EXTERNAL_DNS_DOMAIN=oxide.test
3702024-05-31T06:50:40.558Z+ sed -i~ '
3712024-05-31T06:50:40.558Z /^external_dns_ips/c\
3722024-05-31T06:50:40.558Zexternal_dns_ips = [ "10.151.1.164", "10.151.1.165" ]
3732024-05-31T06:50:40.558Z /^\[\[internal_services_ip_pool_ranges/,/^$/ {
3742024-05-31T06:50:40.558Z /^first/c\
3752024-05-31T06:50:40.558Zfirst = "10.151.1.164"
3762024-05-31T06:50:40.558Z /^last/c\
3772024-05-31T06:50:40.558Zlast = "10.151.1.173"
3782024-05-31T06:50:40.558Z }
3792024-05-31T06:50:40.558Z /^infra_ip_first/c\
3802024-05-31T06:50:40.558Zinfra_ip_first = "10.151.1.174"
3812024-05-31T06:50:40.558Z /^infra_ip_last/c\
3822024-05-31T06:50:40.558Zinfra_ip_last = "10.151.1.174"
3832024-05-31T06:50:40.558Z /^\[\[rack_network_config.ports/,/^$/ {
3842024-05-31T06:50:40.558Z /^routes/c\
3852024-05-31T06:50:40.559Zroutes = \[{nexthop = "10.151.1.100", destination = "0.0.0.0/0"}\]
3862024-05-31T06:50:40.559Z /^addresses/c\
3872024-05-31T06:50:40.559Zaddresses = \[{address = "10.151.1.174/24"} \]
3882024-05-31T06:50:40.559Z }
3892024-05-31T06:50:40.559Z' pkg/config-rss.toml
3902024-05-31T06:50:40.559Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
3912024-05-31T06:50:40.559Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
3922024-05-31T06:50:40.559Z+++ pkg/config-rss.toml Fri May 31 06:50:40 2024
3932024-05-31T06:50:40.559Z@@ -33,7 +33,7 @@
3942024-05-31T06:50:40.559Z # the DNS domain delegated to the rack by the customer. Each of these addresses
3952024-05-31T06:50:40.559Z # must be contained in one of the "internal services" IP Pool ranges listed
3962024-05-31T06:50:40.559Z # below.
3972024-05-31T06:50:40.559Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
3982024-05-31T06:50:40.559Z+external_dns_ips = [ "10.151.1.164", "10.151.1.165" ]
3992024-05-31T06:50:40.559Z
4002024-05-31T06:50:40.559Z # Initial TLS certificates for the external API
4012024-05-31T06:50:40.559Z #
4022024-05-31T06:50:40.559Z@@ -69,8 +69,8 @@
4032024-05-31T06:50:40.559Z #
4042024-05-31T06:50:40.559Z # For more on this and what to put here, see docs/how-to-run.adoc.
4052024-05-31T06:50:40.559Z [[internal_services_ip_pool_ranges]]
4062024-05-31T06:50:40.559Z-first = "192.168.1.20"
4072024-05-31T06:50:40.559Z-last = "192.168.1.29"
4082024-05-31T06:50:40.560Z+first = "10.151.1.164"
4092024-05-31T06:50:40.560Z+last = "10.151.1.173"
4102024-05-31T06:50:40.560Z
4112024-05-31T06:50:40.560Z # TODO - this configuration is subject to change going forward. Ultimately these
4122024-05-31T06:50:40.560Z # parameters should be provided to the control plane via wicket, but we need to
4132024-05-31T06:50:40.560Z@@ -91,8 +91,8 @@
4142024-05-31T06:50:40.560Z # A range of IP addresses used by Boundary Services on the external network. In
4152024-05-31T06:50:40.560Z # a real system, these would be addresses of the uplink ports on the Sidecar.
4162024-05-31T06:50:40.560Z # With softnpu, only one address is used.
4172024-05-31T06:50:40.560Z-infra_ip_first = "192.168.1.30"
4182024-05-31T06:50:40.560Z-infra_ip_last = "192.168.1.30"
4192024-05-31T06:50:40.560Z+infra_ip_first = "10.151.1.174"
4202024-05-31T06:50:40.560Z+infra_ip_last = "10.151.1.174"
4212024-05-31T06:50:40.560Z
4222024-05-31T06:50:40.560Z # Configurations for BGP routers to run on the scrimlets.
4232024-05-31T06:50:40.560Z bgp = []
4242024-05-31T06:50:40.560Z@@ -100,9 +100,9 @@
4252024-05-31T06:50:40.560Z # You can configure multiple uplinks by repeating the following stanza
4262024-05-31T06:50:40.560Z [[rack_network_config.ports]]
4272024-05-31T06:50:40.560Z # Routes associated with this port.
4282024-05-31T06:50:40.560Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
4292024-05-31T06:50:40.560Z+routes = [{nexthop = "10.151.1.100", destination = "0.0.0.0/0"}]
4302024-05-31T06:50:40.560Z # Addresses associated with this port.
4312024-05-31T06:50:40.585Z-addresses = [{address = "192.168.1.30/24"}]
4322024-05-31T06:50:40.585Z+addresses = [{address = "10.151.1.174/24"} ]
4332024-05-31T06:50:40.586Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
4342024-05-31T06:50:40.586Z port = "qsfp0"
4352024-05-31T06:50:40.586Z # The speed of this port.
4362024-05-31T06:50:40.586Z+ true
4372024-05-31T06:50:40.586Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
4382024-05-31T06:50:40.586Za pkg/config-rss.toml 7K
4392024-05-31T06:50:40.586Za pkg/config.toml 4K
4402024-05-31T06:50:40.586Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
4412024-05-31T06:50:40.586Z+ TLS_NAME=recovery.sys.oxide.test
4422024-05-31T06:50:40.586Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
4432024-05-31T06:50:40.875Z.......+......+..+...+...+.+..................+..+......+...+......+....+..............+.+........................+...+..+.+......+........+.......+...+..+.+..+.......+...+........+...+....+...+..................+.....+.+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+............+...+.....+....+......+.....................+........+...+...+.+......+...+............+..............+....+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*........+............+...+.......+..+..........+...+...........+.+....................+....+...........+.+.....+.......+......+.........+..+...............+...+...+.......+.....+.+..+...+..........+...........+......+....+...............+.........+............+.........+...............+...+.....+...+...+.......+..+............+...............+................+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
4442024-05-31T06:50:42.173Z.......+.....+.+..+.........+...+.......+.....+.........+....+............+...+...+...+..+...+.......+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*......+........+......+....+.....+......+....+.....+......+......+...+......................+...+..+....+.........+......+..+...............+...+..........+..+....+........+.+............+...+..+...+.+.....+.+.....+...+......+....+.....+.......+...+..+.........................+..+...+....+...+..+..........+........+...+....+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..............+.........+...+..................+....................+............+...+......+................+............+........+.............+...............+......+.....+....+.........+..+..........+.....................+...+.....................+........+...+...+.......+.........+......+..+...+...............+...+....+...........+....+...+.........+........+...+...+....+.....+.......+...............+......+..+.......+...+.....+......+.....................+..........+.....+...+......+..........+......+......+........+.......+.....+...+...............+......+.........+.........+...+...............+............+.+......+.........+......+......+..+..........+...........+....+......+..+......................+........+....+...+.....+.+..+.............+.....+......+.+.........+...........+..........+.....+...+.........+.+..+...................+......+..................+...........+.+...+...........+......+.......+...............+..+.+..+.......+.....+....+.....................+.....+.+...+......+..............+.......+.....+..........+.................+...+............+...+....+.........+......+...+......+.....+......+.+........+.+.....+...............+....+...........+.+..+.+..+..................+.+......+............+..+...+......+............+...............+...+.+......+.....+...+......+.+.....+............+.+......+......+.....+...+........................+...+...............+....+.....+.........................+...+.....+......+.+...........+....+.......................+.+........................+..+.+..............+...+..........+...............+......+..+.......+..+....+.....+......+.+...............+.....+.......+..+....+.....+...................+.....+..........+...+..+..........+...+..+......+.......+...+.................+....+.........+..+.........+...+...+....+...+..+...................+.....+................+......+........+.........+......+....+...............+...+..+...............+.+......+...+.................+...+...............+...+.......+...+..+.......+......+.....+..........+.........+...+..+....+..+..........+.......................+......+..........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
4452024-05-31T06:50:42.197Z-----
4462024-05-31T06:50:42.198Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
4472024-05-31T06:50:42.224Za pkg/initial-tls-cert.pem 2K
4482024-05-31T06:50:42.224Za pkg/initial-tls-key.pem 4K
4492024-05-31T06:50:42.224Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
4502024-05-31T06:50:42.224Z+ rmdir pkg
4512024-05-31T06:50:42.225Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
4522024-05-31T06:50:42.225Z+ pfexec mkdir -p /usr/oxide
4532024-05-31T06:50:42.225Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v0.1.0
4542024-05-31T06:50:42.897Z+ pfexec chmod +x /usr/oxide/oxide
4552024-05-31T06:50:42.922Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
4562024-05-31T06:51:01.212Z+ OMICRON_NO_UNINSTALL=1
4572024-05-31T06:51:01.212Z+ ptime -m pfexec ./target/release/omicron-package -t test install
4582024-05-31T06:51:01.237ZLogging to: /opt/oxide/work/out/LOG
4592024-05-31T06:51:01.769Z
4602024-05-31T06:51:01.795Zreal 0.551400229
4612024-05-31T06:51:01.795Zuser 0.115425281
4622024-05-31T06:51:01.795Zsys 2.610470677
4632024-05-31T06:51:01.795Ztrap 0.000002990
4642024-05-31T06:51:01.795Ztflt 0.000042849
4652024-05-31T06:51:01.795Zdflt 0.000648153
4662024-05-31T06:51:01.795Zkflt 0.000001840
4672024-05-31T06:51:01.795Zlock 49.803244859
4682024-05-31T06:51:01.795Zslp 0.749472687
4692024-05-31T06:51:01.795Zlat 0.003404723
4702024-05-31T06:51:01.795Zstop 0.209199770
4712024-05-31T06:51:01.795Z+ retry=0
4722024-05-31T06:51:01.795Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4732024-05-31T06:51:01.795Z+ [[ 0 -gt 30 ]]
4742024-05-31T06:51:01.795Z+ sleep 1
4752024-05-31T06:51:02.784Z+ retry=1
4762024-05-31T06:51:02.784Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4772024-05-31T06:51:02.809Z+ [[ 1 -gt 30 ]]
4782024-05-31T06:51:02.809Z+ sleep 1
4792024-05-31T06:51:03.800Z+ retry=2
4802024-05-31T06:51:03.800Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4812024-05-31T06:51:03.825Z+ [[ 2 -gt 30 ]]
4822024-05-31T06:51:03.825Z+ sleep 1
 
4832024-05-31T06:51:04.131Zbuskin console login: May 31 06:51:03 buskin svc.startd[8]: oxide/sled-agent:default failed repeatedly: transitioned to maintenance (see 'svcs -xv' for details)
4842024-05-31T06:51:04.188ZSUNW-MSG-ID: SMF-8000-YX, TYPE: defect, VER: 1, SEVERITY: major
4852024-05-31T06:51:04.242ZEVENT-TIME: Fri May 31 06:51:03 UTC 2024
4862024-05-31T06:51:04.276ZPLATFORM: R152-Z32-00, CSN: GKG8N7212A0021, HOSTNAME: buskin
4872024-05-31T06:51:04.322ZSOURCE: software-diagnosis, REV: 0.1
4882024-05-31T06:51:04.355ZEVENT-ID: 63e672d2-6953-476a-a2ca-c244b60b8f30
4892024-05-31T06:51:04.389ZDESC: A service failed - the instance is restarting too quickly.
4902024-05-31T06:51:04.423ZRefer to http://illumos.org/msg/SMF-8000-YX for more information.
4912024-05-31T06:51:04.456ZAUTO-RESPONSE: The service has been placed into the maintenance state.
4922024-05-31T06:51:04.489ZIMPACT: svc:/oxide/sled-agent:default is unavailable.
4932024-05-31T06:51:04.523ZREC-ACTION: Run 'svcs -xv svc:/oxide/sled-agent:default' to determine the generic reason why the service failed, the location of any logfiles, and a list of other services impacted.
 
4942024-05-31T06:51:04.813Z+ retry=3
4952024-05-31T06:51:04.813Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4962024-05-31T06:51:04.838Z+ [[ 3 -gt 30 ]]
4972024-05-31T06:51:04.838Z+ sleep 1
4982024-05-31T06:51:05.827Z+ retry=4
4992024-05-31T06:51:05.827Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5002024-05-31T06:51:05.852Z+ [[ 4 -gt 30 ]]
5012024-05-31T06:51:05.852Z+ sleep 1
5022024-05-31T06:51:06.842Z+ retry=5
5032024-05-31T06:51:06.842Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5042024-05-31T06:51:06.867Z+ [[ 5 -gt 30 ]]
5052024-05-31T06:51:06.867Z+ sleep 1
5062024-05-31T06:51:07.855Z+ retry=6
5072024-05-31T06:51:07.855Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5082024-05-31T06:51:07.879Z+ [[ 6 -gt 30 ]]
5092024-05-31T06:51:07.879Z+ sleep 1
5102024-05-31T06:51:08.868Z+ retry=7
5112024-05-31T06:51:08.868Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5122024-05-31T06:51:08.892Z+ [[ 7 -gt 30 ]]
5132024-05-31T06:51:08.893Z+ sleep 1
5142024-05-31T06:51:09.882Z+ retry=8
5152024-05-31T06:51:09.882Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5162024-05-31T06:51:09.907Z+ [[ 8 -gt 30 ]]
5172024-05-31T06:51:09.907Z+ sleep 1
5182024-05-31T06:51:10.896Z+ retry=9
5192024-05-31T06:51:10.897Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5202024-05-31T06:51:10.921Z+ [[ 9 -gt 30 ]]
5212024-05-31T06:51:10.921Z+ sleep 1
5222024-05-31T06:51:11.909Z+ retry=10
5232024-05-31T06:51:11.909Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5242024-05-31T06:51:11.934Z+ [[ 10 -gt 30 ]]
5252024-05-31T06:51:11.934Z+ sleep 1
5262024-05-31T06:51:12.924Z+ retry=11
5272024-05-31T06:51:12.924Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5282024-05-31T06:51:12.949Z+ [[ 11 -gt 30 ]]
5292024-05-31T06:51:12.949Z+ sleep 1
5302024-05-31T06:51:13.938Z+ retry=12
5312024-05-31T06:51:13.938Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5322024-05-31T06:51:13.962Z+ [[ 12 -gt 30 ]]
5332024-05-31T06:51:13.962Z+ sleep 1
5342024-05-31T06:51:14.952Z+ retry=13
5352024-05-31T06:51:14.952Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5362024-05-31T06:51:14.976Z+ [[ 13 -gt 30 ]]
5372024-05-31T06:51:14.976Z+ sleep 1
5382024-05-31T06:51:15.964Z+ retry=14
5392024-05-31T06:51:15.964Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5402024-05-31T06:51:15.989Z+ [[ 14 -gt 30 ]]
5412024-05-31T06:51:15.989Z+ sleep 1
5422024-05-31T06:51:16.978Z+ retry=15
5432024-05-31T06:51:16.978Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5442024-05-31T06:51:17.003Z+ [[ 15 -gt 30 ]]
5452024-05-31T06:51:17.003Z+ sleep 1
5462024-05-31T06:51:17.991Z+ retry=16
5472024-05-31T06:51:17.991Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5482024-05-31T06:51:18.016Z+ [[ 16 -gt 30 ]]
5492024-05-31T06:51:18.016Z+ sleep 1
5502024-05-31T06:51:19.006Z+ retry=17
5512024-05-31T06:51:19.006Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5522024-05-31T06:51:19.030Z+ [[ 17 -gt 30 ]]
5532024-05-31T06:51:19.031Z+ sleep 1
5542024-05-31T06:51:20.019Z+ retry=18
5552024-05-31T06:51:20.019Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5562024-05-31T06:51:20.043Z+ [[ 18 -gt 30 ]]
5572024-05-31T06:51:20.043Z+ sleep 1
5582024-05-31T06:51:21.032Z+ retry=19
5592024-05-31T06:51:21.033Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5602024-05-31T06:51:21.060Z+ [[ 19 -gt 30 ]]
5612024-05-31T06:51:21.060Z+ sleep 1
5622024-05-31T06:51:22.043Z+ retry=20
5632024-05-31T06:51:22.043Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5642024-05-31T06:51:22.068Z+ [[ 20 -gt 30 ]]
5652024-05-31T06:51:22.068Z+ sleep 1
5662024-05-31T06:51:23.057Z+ retry=21
5672024-05-31T06:51:23.057Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5682024-05-31T06:51:23.081Z+ [[ 21 -gt 30 ]]
5692024-05-31T06:51:23.081Z+ sleep 1
5702024-05-31T06:51:24.072Z+ retry=22
5712024-05-31T06:51:24.072Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5722024-05-31T06:51:24.096Z+ [[ 22 -gt 30 ]]
5732024-05-31T06:51:24.096Z+ sleep 1
5742024-05-31T06:51:25.084Z+ retry=23
5752024-05-31T06:51:25.085Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5762024-05-31T06:51:25.109Z+ [[ 23 -gt 30 ]]
5772024-05-31T06:51:25.109Z+ sleep 1
5782024-05-31T06:51:26.098Z+ retry=24
5792024-05-31T06:51:26.098Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5802024-05-31T06:51:26.123Z+ [[ 24 -gt 30 ]]
5812024-05-31T06:51:26.123Z+ sleep 1
5822024-05-31T06:51:27.110Z+ retry=25
5832024-05-31T06:51:27.110Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5842024-05-31T06:51:27.135Z+ [[ 25 -gt 30 ]]
5852024-05-31T06:51:27.135Z+ sleep 1
5862024-05-31T06:51:28.124Z+ retry=26
5872024-05-31T06:51:28.124Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5882024-05-31T06:51:28.148Z+ [[ 26 -gt 30 ]]
5892024-05-31T06:51:28.148Z+ sleep 1
5902024-05-31T06:51:29.138Z+ retry=27
5912024-05-31T06:51:29.138Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5922024-05-31T06:51:29.162Z+ [[ 27 -gt 30 ]]
5932024-05-31T06:51:29.162Z+ sleep 1
5942024-05-31T06:51:30.152Z+ retry=28
5952024-05-31T06:51:30.152Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
5962024-05-31T06:51:30.176Z+ [[ 28 -gt 30 ]]
5972024-05-31T06:51:30.177Z+ sleep 1
5982024-05-31T06:51:31.165Z+ retry=29
5992024-05-31T06:51:31.165Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
6002024-05-31T06:51:31.189Z+ [[ 29 -gt 30 ]]
6012024-05-31T06:51:31.190Z+ sleep 1
6022024-05-31T06:51:32.179Z+ retry=30
6032024-05-31T06:51:32.179Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
6042024-05-31T06:51:32.203Z+ [[ 30 -gt 30 ]]
6052024-05-31T06:51:32.203Z+ sleep 1
6062024-05-31T06:51:33.190Z+ retry=31
6072024-05-31T06:51:33.191Z+ curl --head --silent -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
6082024-05-31T06:51:33.215Z+ [[ 31 -gt 30 ]]
6092024-05-31T06:51:33.215Z+ echo 'Failed to reach switch zone after 30 seconds'
6102024-05-31T06:51:33.215Z+ exit 1
6112024-05-31T06:51:33.215ZFailed to reach switch zone after 30 seconds
6122024-05-31T06:51:33.216Z+ _exit_trap
6132024-05-31T06:51:33.216Z+ local status=1
6142024-05-31T06:51:33.216Z+ [[ 1 -eq 0 ]]
6152024-05-31T06:51:33.216Z+ set +o errexit
6162024-05-31T06:51:33.216Z+ set -o xtrace
6172024-05-31T06:51:33.216Z+ banner evidence
6182024-05-31T06:51:33.216Z
6192024-05-31T06:51:33.216Z ###### # # # ##### ###### # # #### ######
6202024-05-31T06:51:33.216Z # # # # # # # ## # # # #
6212024-05-31T06:51:33.216Z ##### # # # # # ##### # # # # #####
6222024-05-31T06:51:33.216Z # # # # # # # # # # # #
6232024-05-31T06:51:33.216Z # # # # # # # # ## # # #
6242024-05-31T06:51:33.216Z ###### ## # ##### ###### # # #### ######
6252024-05-31T06:51:33.216Z
6262024-05-31T06:51:33.216Z+ zoneadm list -civ
6272024-05-31T06:51:33.216Z ID NAME STATUS PATH BRAND IP
6282024-05-31T06:51:33.216Z 0 global running / ipkg shared
6292024-05-31T06:51:33.216Z 1 sidecar_softnpu running /sidecar/sidecar_softnpu omicron1 excl
6302024-05-31T06:51:33.216Z+ pfexec dladm show-phys -m
6312024-05-31T06:51:33.216ZLINK SLOT ADDRESS INUSE CLIENT
6322024-05-31T06:51:33.216Zigb0 primary 18:c0:4d:d:9f:b2 yes igb0
6332024-05-31T06:51:33.216Zigb1 primary 18:c0:4d:d:9f:b3 no --
6342024-05-31T06:51:33.216Ztfpkt0 primary 2:0:0:0:0:0 no --
6352024-05-31T06:51:33.217Z+ pfexec dladm show-link
6362024-05-31T06:51:33.244ZLINK CLASS MTU STATE BRIDGE OVER
6372024-05-31T06:51:33.244Zigb0 phys 1500 up -- --
6382024-05-31T06:51:33.244Zigb1 phys 1500 unknown -- --
6392024-05-31T06:51:33.244Ztfpkt0 phys 1500 down -- --
6402024-05-31T06:51:33.244Znet0 simnet 9000 up -- sc0_0
6412024-05-31T06:51:33.244Zsc0_0 simnet 9000 up -- net0
6422024-05-31T06:51:33.244Znet1 simnet 9000 up -- sc1_0
6432024-05-31T06:51:33.244Zsc1_0 simnet 9000 up -- net1
6442024-05-31T06:51:33.244Zsc0_1 vnic 1500 up -- igb0
6452024-05-31T06:51:33.244Ztfportrear0_0 simnet 9000 up -- sidecar_shi0
6462024-05-31T06:51:33.244Zsidecar_shi0 simnet 9000 up -- tfportrear0_0
6472024-05-31T06:51:33.245Ztfportqsfp0_0 simnet 9000 up -- sidecar_shi1
6482024-05-31T06:51:33.245Zsidecar_shi1 simnet 9000 up -- tfportqsfp0_0
6492024-05-31T06:51:33.245Zbootstrap_stub0 etherstub 9000 up -- --
6502024-05-31T06:51:33.245Zbootstrap0 vnic 1500 up -- bootstrap_stub0
6512024-05-31T06:51:33.245Zunderlay_stub0 etherstub 9000 up -- --
6522024-05-31T06:51:33.245Zunderlay0 vnic 9000 up -- underlay_stub0
6532024-05-31T06:51:33.245Z+ pfexec dladm show-vnic
6542024-05-31T06:51:33.245ZLINK OVER SPEED MACADDRESS MACADDRTYPE VID
6552024-05-31T06:51:33.245Zsc0_1 igb0 1000 a8:e1:de:1:70:1d fixed 0
6562024-05-31T06:51:33.245Zbootstrap0 bootstrap_stub0 0 2:8:20:7f:42:39 random 0
6572024-05-31T06:51:33.245Zunderlay0 underlay_stub0 0 2:8:20:f6:a5:2f random 0
6582024-05-31T06:51:33.245Z+ pfexec ipadm
6592024-05-31T06:51:33.245ZADDROBJ TYPE STATE ADDR
6602024-05-31T06:51:33.245Zlo0/v4 static ok 127.0.0.1/8
6612024-05-31T06:51:33.245Zigb0/dhcp dhcp ok 10.151.1.100/24
6622024-05-31T06:51:33.245Zlo0/v6 static ok ::1/128
6632024-05-31T06:51:33.245Znet0/ll addrconf ok fe80::26:f9ff:fed0:23ad%net0/10
6642024-05-31T06:51:33.245Znet1/ll addrconf ok fe80::60f8:61ff:fe97:46f%net1/10
6652024-05-31T06:51:33.245Zbootstrap0/ll addrconf ok fe80::8:20ff:fe7f:4239%bootstrap0/10
6662024-05-31T06:51:33.245Zbootstrap0/bootstrap6 static ok fdb0:18c0:4d0d:9fb2::1/64
6672024-05-31T06:51:33.245Z+ pfexec netstat -rncva
6682024-05-31T06:51:33.293Z
6692024-05-31T06:51:33.293ZIRE Table: IPv4
6702024-05-31T06:51:33.293Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
6712024-05-31T06:51:33.293Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
6722024-05-31T06:51:33.293Z0.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6732024-05-31T06:51:33.293Zdefault 0.0.0.0 10.151.1.1 igb0 1500 9 UG 1694272 0
6742024-05-31T06:51:33.293Z10.0.0.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6752024-05-31T06:51:33.293Z10.151.1.0 255.255.255.255 -- igb0 1500 1 UHb 0 0
6762024-05-31T06:51:33.293Z10.151.1.0/24 255.255.255.0 10.151.1.100 igb0 1500 3 U 0 0
6772024-05-31T06:51:33.293Z10.151.1.1/32 255.255.255.255 10.151.1.100 igb0 1500 3 UC 0 0
6782024-05-31T06:51:33.293Z10.151.1.100 255.255.255.255 -- igb0 1500 2 UHL 2 3364885
6792024-05-31T06:51:33.293Z10.151.1.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6802024-05-31T06:51:33.294Z10.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 0
6812024-05-31T06:51:33.294Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 56 56
6822024-05-31T06:51:33.294Z255.255.255.255 255.255.255.255 -- igb0 1500 1 UHb 0 1
6832024-05-31T06:51:33.294Z
6842024-05-31T06:51:33.294ZIRE Table: IPv6
6852024-05-31T06:51:33.294Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
6862024-05-31T06:51:33.294Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
6872024-05-31T06:51:33.294Zfdb0:18c0:4d0d:9fb2::1 -- bootstrap0 1500 2 UHL 0 0
6882024-05-31T06:51:33.294Z::1 ::1 lo0 8252 2 UH 28 28
6892024-05-31T06:51:33.294Zfe80::8:20ff:fe7f:4239 -- bootstrap0 1500 2 UHL 0 0
6902024-05-31T06:51:33.294Zfe80::60f8:61ff:fe97:46f -- net1 9000 2 UHL 0 0
6912024-05-31T06:51:33.294Zfe80::26:f9ff:fed0:23ad -- net0 9000 2 UHL 0 0
6922024-05-31T06:51:33.294Zfdb0:18c0:4d0d:9fb2::/64 fdb0:18c0:4d0d:9fb2::1 bootstrap0 1500 2 U 0 0
6932024-05-31T06:51:33.294Zfe80::/10 fe80::8:20ff:fe7f:4239 bootstrap0 1500 2 U 0 0
6942024-05-31T06:51:33.294Zfe80::/10 fe80::60f8:61ff:fe97:46f net1 9000 2 U 0 0
6952024-05-31T06:51:33.294Zfe80::/10 fe80::26:f9ff:fed0:23ad net0 9000 2 U 0 0
6962024-05-31T06:51:33.294Z+ pfexec netstat -anu
6972024-05-31T06:51:33.294Z
6982024-05-31T06:51:33.294ZUDP: IPv4
6992024-05-31T06:51:33.294Z Local Address Remote Address User Pid Command State
7002024-05-31T06:51:33.294Z-------------------- -------------------- -------- ------ -------------- ----------
7012024-05-31T06:51:33.294Z *.* root 78 in.mpathd Unbound
7022024-05-31T06:51:33.295Z *.* root 78 in.mpathd Unbound
7032024-05-31T06:51:33.295Z *.68 root 318 dhcpagent Idle
7042024-05-31T06:51:33.295Z *.546 root 318 dhcpagent Idle
7052024-05-31T06:51:33.295Z10.151.1.100.68 root 318 dhcpagent Idle
7062024-05-31T06:51:33.295Z *.111 daemon 718 rpcbind Idle
7072024-05-31T06:51:33.295Z *.* daemon 718 rpcbind Unbound
7082024-05-31T06:51:33.295Z *.49959 daemon 718 rpcbind Idle
7092024-05-31T06:51:33.295Z *.111 daemon 718 rpcbind Idle
7102024-05-31T06:51:33.295Z *.* daemon 718 rpcbind Unbound
7112024-05-31T06:51:33.295Z *.61345 daemon 718 rpcbind Idle
7122024-05-31T06:51:33.295Z *.* root 728 in.ndpd Unbound
7132024-05-31T06:51:33.295Z *.* root 902 rcm_daemon Unbound
7142024-05-31T06:51:33.295Z *.* root 902 rcm_daemon Unbound
7152024-05-31T06:51:33.295Z
7162024-05-31T06:51:33.295ZUDP: IPv6
7172024-05-31T06:51:33.295Z Local Address Remote Address User Pid Command State If
7182024-05-31T06:51:33.295Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
7192024-05-31T06:51:33.295Z *.* root 78 in.mpathd Unbound
7202024-05-31T06:51:33.295Z *.546 root 318 dhcpagent Idle
7212024-05-31T06:51:33.295Z *.111 daemon 718 rpcbind Idle
7222024-05-31T06:51:33.295Z *.* daemon 718 rpcbind Unbound
7232024-05-31T06:51:33.295Z *.49959 daemon 718 rpcbind Idle
7242024-05-31T06:51:33.295Z *.* root 728 in.ndpd Unbound
7252024-05-31T06:51:33.296Z *.* root 902 rcm_daemon Unbound
7262024-05-31T06:51:33.296Zff02::dd.3549 root 1606 ddmd Idle net1
7272024-05-31T06:51:33.296Zfe80::60f8:61ff:fe97:46f.3549 root 1606 ddmd Idle net1
7282024-05-31T06:51:33.296Zff02::dd.3549 root 1606 ddmd Idle net0
7292024-05-31T06:51:33.296Zfe80::26:f9ff:fed0:23ad.3549 root 1606 ddmd Idle net0
7302024-05-31T06:51:33.296Z
7312024-05-31T06:51:33.296ZTCP: IPv4
7322024-05-31T06:51:33.296Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State
7332024-05-31T06:51:33.296Z-------------------- -------------------- -------- ------ -------------- ------ ------ ------ ------ -----------
7342024-05-31T06:51:33.296Z127.0.0.1.5999 *.* root 78 in.mpathd 0 0 128000 0 LISTEN
7352024-05-31T06:51:33.296Z127.0.0.1.4999 *.* root 318 dhcpagent 0 0 128000 0 LISTEN
7362024-05-31T06:51:33.296Z *.111 *.* daemon 718 rpcbind 0 0 128000 0 LISTEN
7372024-05-31T06:51:33.296Z *.* *.* daemon 718 rpcbind 0 0 128000 0 IDLE
7382024-05-31T06:51:33.296Z *.111 *.* daemon 718 rpcbind 0 0 128000 0 LISTEN
7392024-05-31T06:51:33.296Z *.* *.* daemon 718 rpcbind 0 0 128000 0 IDLE
7402024-05-31T06:51:33.296Z10.151.1.100.63267 54.200.19.125.443 root 809 agent 1049792 0 128872 0 ESTABLISHED
7412024-05-31T06:51:33.296Z10.151.1.100.35038 54.200.19.125.443 root 809 agent 1049792 4708 128872 0 ESTABLISHED
7422024-05-31T06:51:33.296Z10.151.1.100.60066 54.200.19.125.443 1049792 0 128872 0 TIME_WAIT
7432024-05-31T06:51:33.296Z10.151.1.100.58207 54.200.19.125.443 1049792 0 128872 0 TIME_WAIT
7442024-05-31T06:51:33.296Z10.151.1.100.49761 54.200.19.125.443 1049792 0 128872 0 TIME_WAIT
7452024-05-31T06:51:33.296Z10.151.1.100.59401 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
7462024-05-31T06:51:33.296Z10.151.1.100.39008 172.20.0.5.12346 512592 0 128872 0 TIME_WAIT
7472024-05-31T06:51:33.296Z
7482024-05-31T06:51:33.297ZTCP: IPv6
7492024-05-31T06:51:33.297Z Local Address Remote Address User Pid Command Swind Send-Q Rwind Recv-Q State If
7502024-05-31T06:51:33.297Z--------------------------------- --------------------------------- -------- ------ -------------- ------ ------ ------ ------ ----------- -----
7512024-05-31T06:51:33.297Z::1.5999 *.* root 78 in.mpathd 0 0 128000 0 LISTEN
7522024-05-31T06:51:33.297Z *.111 *.* daemon 718 rpcbind 0 0 128000 0 LISTEN
7532024-05-31T06:51:33.297Z *.* *.* daemon 718 rpcbind 0 0 128000 0 IDLE
7542024-05-31T06:51:33.297Z::1.8000 *.* root 1606 ddmd 0 0 128000 0 LISTEN
7552024-05-31T06:51:33.297Z
7562024-05-31T06:51:33.297ZActive UNIX domain sockets
7572024-05-31T06:51:33.297ZAddress Type User Pid Command Local Address Remote Address
7582024-05-31T06:51:33.297Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
7592024-05-31T06:51:33.297Zfffffe85b6b083f0 stream-ord root 1606 ddmd
7602024-05-31T06:51:33.297Zfffffe85b6b083f0 stream-ord root 1606 ddmd
7612024-05-31T06:51:33.297Zfffffe85b6b08790 stream-ord root 1606 ddmd
7622024-05-31T06:51:33.297Zfffffe85873f8758 stream-ord root 809 agent /var/run/buildomat.sock
7632024-05-31T06:51:33.297Zfffffe85873f8af8 stream-ord root 809 agent
7642024-05-31T06:51:33.297Zfffffe85873f8af8 stream-ord root 809 agent
7652024-05-31T06:51:33.297Zfffffe8587336010 stream-ord root 809 agent
7662024-05-31T06:51:33.297Zfffffe85873363b0 stream-ord root 923 zoneadmd /var/run/zones/sidecar_softnpu.console_
7672024-05-31T06:51:33.297Zfffffe8583ffd008 stream-ord root 764 inetd /var/run/.inetd.uds
7682024-05-31T06:51:33.323Zfffffe8583ffd3a8 dgram root 728 in.ndpd /var/run/in.ndpd_mib
7692024-05-31T06:51:33.323Zfffffe8583ffd748 stream-ord root 728 in.ndpd /var/run/in.ndpd_ipadm
7702024-05-31T06:51:33.323Z+ pfexec arp -an
7712024-05-31T06:51:33.323ZNet to Media Table: IPv4
7722024-05-31T06:51:33.323ZDevice IP Address Mask Flags Phys Addr
7732024-05-31T06:51:33.323Z------ -------------------- --------------- -------- ---------------
7742024-05-31T06:51:33.323Zigb0 10.151.1.1 255.255.255.255 aa:00:04:00:ca:fe
7752024-05-31T06:51:33.323Zigb0 10.151.1.100 255.255.255.255 SPLA 18:c0:4d:0d:9f:b2
7762024-05-31T06:51:33.323Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
7772024-05-31T06:51:33.323Zlocal v6:
7782024-05-31T06:51:33.323Zlocal v4:
7792024-05-31T06:51:33.323Zrouter v6:
7802024-05-31T06:51:33.323Zrouter v4_idx:
7812024-05-31T06:51:33.323Zrouter v4_routes:
7822024-05-31T06:51:33.323Zresolver v4:
7832024-05-31T06:51:33.323Z10.151.1.100 -> 18:c0:4d:0d:9f:b2
7842024-05-31T06:51:33.323Zresolver v6:
7852024-05-31T06:51:33.323Znat v4:
7862024-05-31T06:51:33.323Znat_v6:
7872024-05-31T06:51:33.323Zport_mac:
7882024-05-31T06:51:33.323Zproxy arp:
7892024-05-31T06:51:33.324Z10.151.1.164/10.151.1.227: a8:e1:de:01:70:1d
7902024-05-31T06:51:33.324Z+ pfexec /opt/oxide/opte/bin/opteadm list-ports
7912024-05-31T06:51:33.324ZLINK MAC ADDRESS IPv4 ADDRESS EPHEMERAL IPv4 FLOATING IPv4 IPv6 ADDRESS EXTERNAL IPv6 FLOATING IPv6 STATE
7922024-05-31T06:51:33.324Z+ pfexec /opt/oxide/opte/bin/opteadm dump-v2b
7932024-05-31T06:51:33.324ZVirtual to Boundary Mappings
7942024-05-31T06:51:33.324Z======================================================================
7952024-05-31T06:51:33.324Z
7962024-05-31T06:51:33.324ZIPv4 mappings
7972024-05-31T06:51:33.324Z----------------------------------------------------------------------
7982024-05-31T06:51:33.324ZTUNNELED PREFIX BOUNDARY IP VNI
7992024-05-31T06:51:33.324Z
8002024-05-31T06:51:33.324ZIPv6 mappings
8012024-05-31T06:51:33.324Z----------------------------------------------------------------------
8022024-05-31T06:51:33.324ZTUNNELED PREFIX BOUNDARY IP VNI
8032024-05-31T06:51:33.324Z
8042024-05-31T06:51:33.324Z+ z_swadm link ls
8052024-05-31T06:51:33.324Z+ echo '== swadm link' ls
8062024-05-31T06:51:33.324Z== swadm link ls
8072024-05-31T06:51:33.324Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm link ls
8082024-05-31T06:51:33.324Zzlogin: zone 'oxz_switch' unknown
8092024-05-31T06:51:33.324Z+ z_swadm addr list
8102024-05-31T06:51:33.324Z+ echo '== swadm addr' list
8112024-05-31T06:51:33.324Z== swadm addr list
8122024-05-31T06:51:33.324Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm addr list
8132024-05-31T06:51:33.325Zzlogin: zone 'oxz_switch' unknown
8142024-05-31T06:51:33.325Z+ z_swadm route list
8152024-05-31T06:51:33.325Z+ echo '== swadm route' list
8162024-05-31T06:51:33.325Z== swadm route list
8172024-05-31T06:51:33.325Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm route list
8182024-05-31T06:51:33.325Zzlogin: zone 'oxz_switch' unknown
8192024-05-31T06:51:33.325Z+ z_swadm arp list
8202024-05-31T06:51:33.325Z+ echo '== swadm arp' list
8212024-05-31T06:51:33.325Z== swadm arp list
8222024-05-31T06:51:33.325Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm arp list
8232024-05-31T06:51:33.325Zzlogin: zone 'oxz_switch' unknown
8242024-05-31T06:51:33.325Z+ z_swadm nat list
8252024-05-31T06:51:33.325Z+ echo '== swadm nat' list
8262024-05-31T06:51:33.325Z== swadm nat list
8272024-05-31T06:51:33.325Z+ pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm nat list
8282024-05-31T06:51:33.325Zzlogin: zone 'oxz_switch' unknown
8292024-05-31T06:51:33.325Z++ pfexec /opt/oxide/opte/bin/opteadm list-ports
8302024-05-31T06:51:33.325Z++ tail +2
8312024-05-31T06:51:33.325Z++ awk '{ print $1; }'
8322024-05-31T06:51:33.350Z+ PORTS=
8332024-05-31T06:51:33.351Z+ pfexec zfs list
8342024-05-31T06:51:33.351ZNAME USED AVAIL REFER MOUNTPOINT
8352024-05-31T06:51:33.351Zrpool 2.11G 5.15G 24K /rpool
8362024-05-31T06:51:33.351Zrpool/home 25.5K 5.15G 25.5K /home
8372024-05-31T06:51:33.351Zrpool/input 2.04G 5.15G 2.04G /input
8382024-05-31T06:51:33.351Zrpool/sidecar 74.9M 5.15G 24K /sidecar
8392024-05-31T06:51:33.351Zrpool/sidecar/sidecar_softnpu 74.9M 5.15G 74.9M /sidecar/sidecar_softnpu
8402024-05-31T06:51:33.351Zrpool/work 24K 5.15G 24K /work
8412024-05-31T06:51:33.351Zrpool/zone 24K 5.15G 24K /zone
8422024-05-31T06:51:33.351Zscratch 116K 1.80T 25K /scratch
8432024-05-31T06:51:33.351Z+ pfexec zpool list
8442024-05-31T06:51:33.351ZNAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
8452024-05-31T06:51:33.351Zrpool 7.50G 2.11G 5.39G - - 0% 28% 1.00x ONLINE -
8462024-05-31T06:51:33.351Zscratch 1.86T 116K 1.86T - - 0% 0% 1.00x ONLINE -
8472024-05-31T06:51:33.351Z+ pfexec fmdump -eVp
8482024-05-31T06:51:33.351ZTIME CLASS
8492024-05-31T06:51:33.351Z+ pfexec ptree -z global
8502024-05-31T06:51:33.351Z8 /lib/svc/bin/svc.startd
8512024-05-31T06:51:33.351Z 646 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
8522024-05-31T06:51:33.351Z 658 /usr/lib/saf/sac -t 300
8532024-05-31T06:51:33.351Z 661 /usr/lib/saf/ttymon
8542024-05-31T06:51:33.352Z 808 /usr/bin/ctrun -l child -o noorphan,regent /opt/buildomat/lib/agent ru
8552024-05-31T06:51:33.352Z 809 /opt/buildomat/lib/agent run
8562024-05-31T06:51:33.352Z 823 /bin/bash /tmp/f5ZMnAoLeEXHexCT.sh
8572024-05-31T06:51:33.352Z 1810 ptree -z global
8582024-05-31T06:51:33.352Z10 /lib/svc/bin/svc.configd
8592024-05-31T06:51:33.352Z47 /lib/inet/ipmgmtd
8602024-05-31T06:51:33.352Z50 /lib/inet/netcfgd
8612024-05-31T06:51:33.352Z53 /sbin/dlmgmtd
8622024-05-31T06:51:33.352Z78 /lib/inet/in.mpathd
8632024-05-31T06:51:33.352Z98 /usr/lib/pfexecd
8642024-05-31T06:51:33.352Z157 /usr/lib/zones/zonestatd
8652024-05-31T06:51:33.352Z197 /usr/lib/sysevent/syseventd
8662024-05-31T06:51:33.352Z210 devfsadmd
8672024-05-31T06:51:33.352Z318 /sbin/dhcpagent
8682024-05-31T06:51:33.352Z571 /usr/lib/picl/picld
8692024-05-31T06:51:33.352Z604 /usr/sbin/nscd
8702024-05-31T06:51:33.352Z645 /usr/lib/utmpd
8712024-05-31T06:51:33.352Z672 /usr/sbin/cron
8722024-05-31T06:51:33.352Z679 /usr/lib/hotplugd
8732024-05-31T06:51:33.352Z688 /usr/sbin/syslogd
8742024-05-31T06:51:33.352Z718 /usr/sbin/rpcbind
8752024-05-31T06:51:33.352Z728 /usr/lib/inet/in.ndpd
8762024-05-31T06:51:33.352Z730 /usr/lib/fm/fmd/fmd
8772024-05-31T06:51:33.353Z764 /usr/lib/inet/inetd start
8782024-05-31T06:51:33.353Z789 /usr/perl5/bin/perl /usr/lib/intrd
8792024-05-31T06:51:33.353Z902 /usr/lib/rcm/rcm_daemon
8802024-05-31T06:51:33.353Z923 zoneadmd -z sidecar_softnpu
8812024-05-31T06:51:33.353Z1604 ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_scrip
8822024-05-31T06:51:33.353Z 1606 /opt/oxide/mg-ddm/ddmd --with-stats --admin-port 8000 --admin-addr ::1
8832024-05-31T06:51:33.353Z+ pfexec svcs -xv
8842024-05-31T06:51:33.424Zsvc:/oxide/sled-agent:default (Oxide Sled Agent)
8852024-05-31T06:51:33.424Z State: maintenance since May 31, 2024 at 06:51:03 AM UTC
8862024-05-31T06:51:33.424ZReason: Restarting too quickly.
8872024-05-31T06:51:33.424Z See: http://illumos.org/msg/SMF-8000-L5
8882024-05-31T06:51:33.424Z See: /var/svc/log/oxide-sled-agent:default.log
8892024-05-31T06:51:33.424ZImpact: This service is not running.
8902024-05-31T06:51:33.449Z++ zoneadm list -n
8912024-05-31T06:51:33.449Z+ for z in $(zoneadm list -n)
8922024-05-31T06:51:33.449Z+ banner sidecar_softnpu
8932024-05-31T06:51:33.449Z
8942024-05-31T06:51:33.449Z #### # ##### ###### #### ## ##### #### ####
8952024-05-31T06:51:33.449Z # # # # # # # # # # # # # #
8962024-05-31T06:51:33.449Z #### # # # ##### # # # # # #### # #
8972024-05-31T06:51:33.449Z # # # # # # ###### ##### # # #
8982024-05-31T06:51:33.449Z # # # # # # # # # # # # # # # #
8992024-05-31T06:51:33.449Z #### # ##### ###### #### # # # # ####### #### ####
9002024-05-31T06:51:33.449Z
9012024-05-31T06:51:33.449Z+ pfexec svcs -xv -z sidecar_softnpu
9022024-05-31T06:51:33.493Z+ pfexec ptree -z sidecar_softnpu
9032024-05-31T06:51:33.518Z926 zsched
9042024-05-31T06:51:33.518Z 943 /sbin/init
9052024-05-31T06:51:33.518Z948 /lib/svc/bin/svc.startd
9062024-05-31T06:51:33.518Z 1474 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -
9072024-05-31T06:51:33.518Z950 /lib/svc/bin/svc.configd
9082024-05-31T06:51:33.518Z994 /lib/inet/netcfgd
9092024-05-31T06:51:33.518Z995 /lib/inet/ipmgmtd
9102024-05-31T06:51:33.518Z1193 /usr/lib/fm/fmd/fmd
9112024-05-31T06:51:33.518Z1287 /usr/lib/pfexecd
9122024-05-31T06:51:33.518Z1322 /softnpu/softnpu --uds-path /softnpu /softnpu/softnpu.toml
9132024-05-31T06:51:33.518Z1423 /usr/sbin/nscd
9142024-05-31T06:51:33.518Z1473 /usr/lib/utmpd
9152024-05-31T06:51:33.518Z1493 /usr/sbin/syslogd
9162024-05-31T06:51:33.519Z1496 /usr/sbin/cron
9172024-05-31T06:51:33.519Z1546 /usr/lib/inet/in.ndpd
9182024-05-31T06:51:33.519Z+ pfexec zlogin sidecar_softnpu ipadm
9192024-05-31T06:51:33.543ZADDROBJ TYPE STATE ADDR
9202024-05-31T06:51:33.543Zlo0/v4 static ok 127.0.0.1/8
9212024-05-31T06:51:33.543Zlo0/v6 static ok ::1/128
9222024-05-31T06:51:33.543Z+ pfexec zlogin sidecar_softnpu netstat -rncva
9232024-05-31T06:51:33.543Z
9242024-05-31T06:51:33.543ZIRE Table: IPv4
9252024-05-31T06:51:33.543Z Destination Mask Gateway Device MTU Ref Flg Out In/Fwd
9262024-05-31T06:51:33.544Z-------------------- --------------- -------------------- ------ ----- --- --- ------ ------
9272024-05-31T06:51:33.544Z127.0.0.1 255.255.255.255 127.0.0.1 lo0 8232 2 UH 0 0
9282024-05-31T06:51:33.544Z
9292024-05-31T06:51:33.544ZIRE Table: IPv6
9302024-05-31T06:51:33.544Z Destination/Mask Gateway If MTU Ref Flags Out In/Fwd
9312024-05-31T06:51:33.544Z--------------------------- --------------------------- ----- ----- --- ----- ------ ------
9322024-05-31T06:51:33.544Z::1 ::1 lo0 8252 2 UH 0 0
9332024-05-31T06:51:33.544Z+ pfexec zlogin sidecar_softnpu netstat -anu
9342024-05-31T06:51:33.569Z
9352024-05-31T06:51:33.569ZUDP: IPv4
9362024-05-31T06:51:33.569Z Local Address Remote Address User Pid Command State
9372024-05-31T06:51:33.569Z-------------------- -------------------- -------- ------ -------------- ----------
9382024-05-31T06:51:33.569Z *.* root 1546 in.ndpd Unbound
9392024-05-31T06:51:33.569Z
9402024-05-31T06:51:33.569ZUDP: IPv6
9412024-05-31T06:51:33.569Z Local Address Remote Address User Pid Command State If
9422024-05-31T06:51:33.569Z--------------------------------- --------------------------------- -------- ------ -------------- ---------- -----
9432024-05-31T06:51:33.569Z *.* root 1546 in.ndpd Unbound
9442024-05-31T06:51:33.569Z
9452024-05-31T06:51:33.570ZActive UNIX domain sockets
9462024-05-31T06:51:33.570ZAddress Type User Pid Command Local Address Remote Address
9472024-05-31T06:51:33.570Z---------------- ---------- -------- ------ -------------- --------------------------------------- ---------------------------------------
9482024-05-31T06:51:33.570Zfffffe85a8cab040 dgram root 1546 in.ndpd /var/run/in.ndpd_mib
9492024-05-31T06:51:33.570Zfffffe85a8cab3e0 stream-ord root 1546 in.ndpd /var/run/in.ndpd_ipadm
9502024-05-31T06:51:33.570Zfffffe85a1302030 dgram root 1322 softnpu /softnpu/server
9512024-05-31T06:51:33.570Zfffffe85a13023d0 stream-ord root 1322 softnpu
9522024-05-31T06:51:33.570Zfffffe85a13023d0 stream-ord root 1322 softnpu
9532024-05-31T06:51:33.570Zfffffe85a1302770 stream-ord root 1322 softnpu
9542024-05-31T06:51:33.570Z+ pfexec zlogin sidecar_softnpu arp -an
9552024-05-31T06:51:33.594Z++ zoneadm list -n
9562024-05-31T06:51:33.594Z++ grep oxz_ntp
9572024-05-31T06:51:33.594Z+ pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log
9582024-05-31T06:51:33.619ZConfig {
9592024-05-31T06:51:33.619Z p4_program: "/softnpu/asic_program.so",
9602024-05-31T06:51:33.619Z ports: [
9612024-05-31T06:51:33.619Z Port {
9622024-05-31T06:51:33.619Z sidecar: "sc0_0",
9632024-05-31T06:51:33.619Z scrimlet: "sidecar_shi0",
9642024-05-31T06:51:33.619Z mtu: 9000,
9652024-05-31T06:51:33.619Z },
9662024-05-31T06:51:33.619Z Port {
9672024-05-31T06:51:33.619Z sidecar: "sc0_1",
9682024-05-31T06:51:33.619Z scrimlet: "sidecar_shi1",
9692024-05-31T06:51:33.619Z mtu: 9000,
9702024-05-31T06:51:33.619Z },
9712024-05-31T06:51:33.619Z ],
9722024-05-31T06:51:33.619Z}
9732024-05-31T06:51:33.619Z+ exit 1
9742024-05-31T06:51:33.619Zprocess exited: duration 78312 ms, exit code 1
 
9752024-05-31T06:51:33.671Zfound 1 output files
9762024-05-31T06:51:33.671Zuploading: /var/svc/log/oxide-sled-agent:default.log (11355 bytes)
9772024-05-31T06:51:34.791Zuploaded: /var/svc/log/oxide-sled-agent:default.log