01KYWFE74RPAMWW5R6S9XCV3QE: helios / deploy

BasicConfig {
    output_rules: [
        "%/var/svc/log/oxide-*.log*",
        "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/system-illumos-*.log*",
        "%/pool/ext/*/crypt/zone/oxz_ntp_*/root/var/log/chrony/*.log*",
        "!/pool/ext/*/crypt/zone/oxz_propolis-server_*/root/var/svc/log/*.log*",
        "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
        "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
        "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
        "/tmp/kstat/*.kstat",
    ],
    rust_toolchain: None,
    target: Some(
        "lab-3.0-opte-0.41",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KYWFF1PVPMK75NK2901QQK22

Tags:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-31T16:36:51.459Zjob dependencies complete; ready to run (waiting for 19 m 13 s)
22026-07-31T16:37:46.593Zjob assigned to worker 01KYWGJ90FJ0HWE5P73JN5PPZ9 [factory lab, sock/17748] (queued for 55 s)
32026-07-31T16:37:46.932Zhttp://172.20.0.5:9969/ipxe/sock... ok
42026-07-31T16:37:46.985Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/kernel/amd64/unix... ok
52026-07-31T16:37:47.032Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive... ok
62026-07-31T16:37:47.066Zhttp://172.20.0.5:9969/os/sock/platform/i86pc/amd64/boot_archive.hash... ok
72026-07-31T16:37:47.099Zhttp://172.20.0.5:9969/postboot/sock... ok
82026-07-31T16:37:47.132ZwcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U wcnCroia6U
92026-07-31T16:37:47.166ZOxide Helios Version helios-3.0.24018 64-bit
102026-07-31T16:37:47.200ZConfiguring devices.
112026-07-31T16:37:47.233Z##################################################
122026-07-31T16:37:47.266Z#### ############# ##
132026-07-31T16:37:47.300Z### ### ############ ##
142026-07-31T16:37:47.333Z## ### # ## ### ## Oxide Computer Company ##
152026-07-31T16:37:47.366Z## ## ## ### # ### ##
162026-07-31T16:37:47.400Z## # ### #### #### This Station Under ##
172026-07-31T16:37:47.433Z### ### #### # ### Computer Control ##
182026-07-31T16:37:47.467Z#### #### ### ## ##
192026-07-31T16:37:47.500Z##################################################
202026-07-31T16:37:47.534ZHostname: sock
212026-07-31T16:37:47.567ZLoading smf(7) service descriptions:
222026-07-31T16:37:48.215Zdownloading input: /input/package/work/package.tar.gz
232026-07-31T16:45:45.992Zdownloaded input: /input/package/work/package.tar.gz
 
242026-07-31T16:45:46.017Zstarting task 0: "setup"
252026-07-31T16:45:46.017Z++ uname -s
262026-07-31T16:45:46.017Z+ kern=SunOS
272026-07-31T16:45:46.017Z+ build_user=build
282026-07-31T16:45:46.017Z+ build_uid=12345
292026-07-31T16:45:46.017Z+ work_dir=/work
302026-07-31T16:45:46.017Z+ input_dir=/input
312026-07-31T16:45:46.017Z+ [[ 0 == 12345 ]]
322026-07-31T16:45:46.017Z+ case "$kern" in
332026-07-31T16:45:46.017Z+ groupadd -g 12345 build
342026-07-31T16:45:46.018Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
352026-07-31T16:45:48.012Z+ zfs create -o mountpoint=/work rpool/work
362026-07-31T16:45:48.036Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
372026-07-31T16:45:48.036Z+ home_fs=zfs
382026-07-31T16:45:48.037Z+ [[ zfs == autofs ]]
392026-07-31T16:45:48.037Z+ mkdir -p /home/build
402026-07-31T16:45:48.061Z+ chown build:build /home/build /work
412026-07-31T16:45:49.040Z+ chmod 0700 /home/build /work
422026-07-31T16:45:49.065Zprocess exited: duration 3052 ms, exit code 0
 
432026-07-31T16:45:49.113Zstarting task 1: "authentication"
442026-07-31T16:45:49.163Zprocess exited: duration 48 ms, exit code 0
 
452026-07-31T16:45:49.212Zstarting task 2: "build"
462026-07-31T16:45:49.236Z+ trap _exit_trap EXIT
472026-07-31T16:45:49.236Z+ pfexec mdb -kwe 'secpolicy_fs_linkdir/v 55 48 89 e5 b8 01 00 00 00 5d c3'
482026-07-31T16:45:49.329Zsecpolicy_fs_linkdir: 0x55 = 0x55
492026-07-31T16:45:49.329Zsecpolicy_fs_linkdir+1: 0xb8 = 0x48
502026-07-31T16:45:49.329Zsecpolicy_fs_linkdir+2: 0x1 = 0x89
512026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+3: 0 = 0xe5
522026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+4: 0 = 0xb8
532026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+5: 0 = 0x1
542026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+6: 0x48 = 0x0
552026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+7: 0x89 = 0x0
562026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+8: 0xe5 = 0x0
572026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+9: 0x48 = 0x5d
582026-07-31T16:45:49.330Zsecpolicy_fs_linkdir+0xa: 0x83 = 0xc3
592026-07-31T16:45:49.354Z+ [[ -d /opt/oxide ]]
602026-07-31T16:45:49.354Z+ pfexec mv /opt/oxide /opt/oxide-underneath
612026-07-31T16:45:49.354Z+ pfexec mkdir /opt/oxide
622026-07-31T16:45:49.354Z+ pfexec mount -F tmpfs -O swap /opt/oxide
632026-07-31T16:45:49.354Z+ [[ -d /opt/oxide-underneath ]]
642026-07-31T16:45:49.354Z+ cd /opt/oxide-underneath
652026-07-31T16:45:49.354Z+ pfexec tar ceEp@/f - .
662026-07-31T16:45:49.354Z+ cd /opt/oxide
672026-07-31T16:45:49.354Z+ pfexec tar xveEp@/f -
682026-07-31T16:45:49.402Zx ., 0 bytes, 0 tape blocks
692026-07-31T16:45:49.402Zx ./opte, 0 bytes, 0 tape blocks
702026-07-31T16:45:49.402Zx ./opte/bin, 0 bytes, 0 tape blocks
712026-07-31T16:45:49.402Zx ./opte/bin/opteadm, 24740168 bytes, 48321 tape blocks
722026-07-31T16:45:49.426Z+ pfexec rm -rf /opt/oxide-underneath
732026-07-31T16:45:49.426Z+ pfexec /sbin/zfs create -o mountpoint=/zone rpool/zone
742026-07-31T16:45:49.450Z+ pfexec mkdir /opt/oxide/work
752026-07-31T16:45:49.450Z+ pfexec chown build:build /opt/oxide/work
762026-07-31T16:45:49.450Z+ cd /opt/oxide/work
772026-07-31T16:45:49.450Z+ ptime -m tar xvzf /input/package/work/package.tar.gz
782026-07-31T16:46:05.447ZDecompressing '/input/package/work/package.tar.gz' with '/usr/bin/gzcat'...
792026-07-31T16:46:05.447Zx .github/buildomat/ci-env.sh, 785 bytes, 2 tape blocks
802026-07-31T16:46:05.447Zx out/target/test, 107 bytes, 1 tape blocks
812026-07-31T16:46:05.447Zx out/npuzone/npuzone, 28059200 bytes, 54804 tape blocks
822026-07-31T16:46:05.447Zx package-manifest.toml, 35397 bytes, 70 tape blocks
832026-07-31T16:46:05.447Zx smf/sled-agent/non-gimlet/config.toml, 5423 bytes, 11 tape blocks
842026-07-31T16:46:05.447Zx target/release/omicron-package, 46019432 bytes, 89882 tape blocks
852026-07-31T16:46:05.447Zx target/release/xtask, 12003048 bytes, 23444 tape blocks
862026-07-31T16:46:05.447Zx target/debug/bootstrap, 416915456 bytes, 814288 tape blocks
872026-07-31T16:46:05.447Zx tests/bootstrap-7bba8094af8d179c, 35342016 bytes, 69028 tape blocks
882026-07-31T16:46:05.447Zx tests/commtest-9378d17b3096c345, 35732848 bytes, 69791 tape blocks
892026-07-31T16:46:05.447Zx tests/dhcp_server-deeac456ecab1631, 35342056 bytes, 69028 tape blocks
902026-07-31T16:46:05.447Zx tests/end_to_end_tests-e0b3d4b2c76394dd, 481413216 bytes, 940261 tape blocks
912026-07-31T16:46:05.447Zx tools/opte_version, 9 bytes, 1 tape blocks
922026-07-31T16:46:05.447Zx tools/opte_version_override, 1245 bytes, 3 tape blocks
932026-07-31T16:46:05.448Zx out/clickhouse.tar.gz, 323659548 bytes, 632148 tape blocks
942026-07-31T16:46:05.448Zx out/clickhouse_keeper.tar.gz, 309536789 bytes, 604565 tape blocks
952026-07-31T16:46:05.448Zx out/clickhouse_server.tar.gz, 323656403 bytes, 632142 tape blocks
962026-07-31T16:46:05.448Zx out/cockroachdb.tar.gz, 167199232 bytes, 326561 tape blocks
972026-07-31T16:46:05.448Zx out/crucible-pantry-zone.tar.gz, 40462519 bytes, 79029 tape blocks
982026-07-31T16:46:05.448Zx out/crucible-utils.tar, 19353600 bytes, 37800 tape blocks
992026-07-31T16:46:05.448Zx out/crucible-zone.tar.gz, 55510607 bytes, 108420 tape blocks
1002026-07-31T16:46:05.448Zx out/external-dns.tar.gz, 49259751 bytes, 96211 tape blocks
1012026-07-31T16:46:05.448Zx out/internal-dns.tar.gz, 49259349 bytes, 96210 tape blocks
1022026-07-31T16:46:05.448Zx out/mg-ddm-gz.tar, 103339008 bytes, 201834 tape blocks
1032026-07-31T16:46:05.448Zx out/nexus.tar.gz, 363383995 bytes, 709735 tape blocks
1042026-07-31T16:46:05.448Zx out/ntp.tar.gz, 31413491 bytes, 61355 tape blocks
1052026-07-31T16:46:05.448Zx out/omicron-gateway-softnpu.tar.gz, 30146469 bytes, 58880 tape blocks
1062026-07-31T16:46:05.448Zx out/omicron-sled-agent.tar, 225664000 bytes, 440750 tape blocks
1072026-07-31T16:46:05.448Zx out/overlay.tar.gz, 3266 bytes, 7 tape blocks
1082026-07-31T16:46:05.448Zx out/oximeter.tar.gz, 66610378 bytes, 130099 tape blocks
1092026-07-31T16:46:05.448Zx out/oxlog.tar, 4100608 bytes, 8009 tape blocks
1102026-07-31T16:46:05.448Zx out/probe.tar.gz, 3109568 bytes, 6074 tape blocks
1112026-07-31T16:46:05.448Zx out/propolis-server.tar.gz, 35196488 bytes, 68744 tape blocks
1122026-07-31T16:46:05.448Zx out/pumpkind-gz.tar, 27286528 bytes, 53294 tape blocks
1132026-07-31T16:46:05.448Zx out/switch-softnpu.tar.gz, 511810516 bytes, 999630 tape blocks
1142026-07-31T16:46:05.472Z
1152026-07-31T16:46:05.473Zreal 15.983450896
1162026-07-31T16:46:05.473Zuser 13.273739248
1172026-07-31T16:46:05.473Zsys 4.739337221
1182026-07-31T16:46:05.473Ztrap 0.000709073
1192026-07-31T16:46:05.473Ztflt 0.000000000
1202026-07-31T16:46:05.473Zdflt 0.000000000
1212026-07-31T16:46:05.473Zkflt 0.000000000
1222026-07-31T16:46:05.473Zlock 0.000000000
1232026-07-31T16:46:05.473Zslp 13.276595073
1242026-07-31T16:46:05.473Zlat 0.674351259
1252026-07-31T16:46:05.473Zstop 0.000136242
1262026-07-31T16:46:05.473Z+ source .github/buildomat/ci-env.sh
1272026-07-31T16:46:05.473Z++ export CARGO_TERM_COLOR=always
1282026-07-31T16:46:05.473Z++ CARGO_TERM_COLOR=always
1292026-07-31T16:46:05.473Z+++ uname -s
1302026-07-31T16:46:05.473Z++ [[ SunOS == \S\u\n\O\S ]]
1312026-07-31T16:46:05.473Z++ coreadm -p /var/tmp/omicron_tmp/core.%f.%p.core
1322026-07-31T16:46:05.473Z+ source tools/opte_version_override
1332026-07-31T16:46:05.473Z++ OPTE_COMMIT=
1342026-07-31T16:46:05.473Z+ [[ x != \x ]]
1352026-07-31T16:46:05.473Z+ bmat address ls
1362026-07-31T16:46:05.473ZNAME CIDR FIRST LAST COUNT
1372026-07-31T16:46:05.473Zextra 10.151.2.0/24 10.151.2.164 10.151.2.227 64
1382026-07-31T16:46:05.473Z++ bmat address ls -f extra -Ho first,last,count
1392026-07-31T16:46:05.473Z+ set -- 10.151.2.164 10.151.2.227 64
1402026-07-31T16:46:05.473Z+ EXTRA_IP_START=10.151.2.164
1412026-07-31T16:46:05.473Z+ EXTRA_IP_END=10.151.2.227
1422026-07-31T16:46:05.473Z+ EXTRA_IP_COUNT=64
1432026-07-31T16:46:05.473Z+ (( EXTRA_IP_COUNT >= 32 ))
1442026-07-31T16:46:05.473Z+ EXTRA_IP_BASE=10.151.2
1452026-07-31T16:46:05.473Z+ EXTRA_IP_FOCTET=164
1462026-07-31T16:46:05.473Z+ EXTRA_IP_LOCTET=227
1472026-07-31T16:46:05.473Z+ SERVICE_IP_POOL_START=10.151.2.164
1482026-07-31T16:46:05.473Z+ SERVICE_IP_POOL_END=10.151.2.173
1492026-07-31T16:46:05.473Z+ DNS_IP1=10.151.2.164
1502026-07-31T16:46:05.473Z+ DNS_IP2=10.151.2.165
1512026-07-31T16:46:05.473Z+ UPLINK_IP=10.151.2.174
1522026-07-31T16:46:05.473Z+ IPPOOL_START=10.151.2.175
1532026-07-31T16:46:05.474Z+ IPPOOL_END=10.151.2.227
1542026-07-31T16:46:05.474Z++ ipadm show-addr -po type,addr
1552026-07-31T16:46:05.474Z++ awk '-F[:/]' '$1 == "dhcp" {print $2}'
1562026-07-31T16:46:05.474Z+ GATEWAY_IP=10.151.2.100
1572026-07-31T16:46:05.474Z+ [[ -n 10.151.2.100 ]]
1582026-07-31T16:46:05.474Z+ ping -s 10.151.2.100 56 1
1592026-07-31T16:46:05.474ZPING 10.151.2.100: 56 data bytes
1602026-07-31T16:46:05.498Z64 bytes from 10.151.2.100: icmp_seq=0. time=0.013 ms
1612026-07-31T16:46:05.498Z
1622026-07-31T16:46:05.498Z----10.151.2.100 PING Statistics----
1632026-07-31T16:46:05.498Z1 packets transmitted, 1 packets received, 0% packet loss
1642026-07-31T16:46:05.498Zround-trip (ms) min/avg/max/stddev = 0.013/0.013/0.013/-nan
1652026-07-31T16:46:05.498Z++ arp -an
1662026-07-31T16:46:05.498Z++ awk -vgw=10.151.2.100 '$2 == gw {print $NF}'
1672026-07-31T16:46:05.498Z+ GATEWAY_MAC=18:c0:4d:0d:a0:2a
1682026-07-31T16:46:05.498Z+ routeadm -e ipv4-forwarding -u
1692026-07-31T16:46:05.592Z+ PXA_START=10.151.2.164
1702026-07-31T16:46:05.592Z+ PXA_END=10.151.2.227
1712026-07-31T16:46:05.592Z+ DISKS=($(pfexec nvmeadm list -p -o disk))
1722026-07-31T16:46:05.616Z++ pfexec nvmeadm list -p -o disk
1732026-07-31T16:46:05.616Z+ pfexec zpool create -f scratch c1t0026B728281787A0d0 c2t0026B72828178F30d0
1742026-07-31T16:46:05.794Z+ ptime -m pfexec ./target/release/xtask virtual-hardware --vdev-dir /scratch create --gateway-ip 10.151.2.100 --gateway-mac 18:c0:4d:0d:a0:2a --pxa-start 10.151.2.164 --pxa-end 10.151.2.227
1752026-07-31T16:46:05.818ZCouldn't find Cargo.toml, using /opt/oxide/work as workspace root
1762026-07-31T16:46:05.867Zcreating virtual hardware
1772026-07-31T16:46:05.891Zcreating /scratch/m2_0.vdev
1782026-07-31T16:46:05.891Zcreating /scratch/m2_1.vdev
1792026-07-31T16:46:05.891Zcreating /scratch/u2_0.vdev
1802026-07-31T16:46:05.891Zcreating /scratch/u2_1.vdev
1812026-07-31T16:46:05.891Zcreating /scratch/u2_2.vdev
1822026-07-31T16:46:05.891Zcreating /scratch/u2_3.vdev
1832026-07-31T16:46:05.891Zcreating /scratch/u2_4.vdev
1842026-07-31T16:46:05.891Zcreating /scratch/u2_5.vdev
1852026-07-31T16:46:05.891Zcreating /scratch/u2_6.vdev
1862026-07-31T16:46:05.891Zcreating /scratch/u2_7.vdev
1872026-07-31T16:46:05.891Zcreating /scratch/u2_8.vdev
1882026-07-31T16:46:05.915ZSimnet net0/sc0_0 exists
1892026-07-31T16:46:05.940ZSimnet net1/sc1_0 exists
1902026-07-31T16:46:05.940ZVnic sc0_1 exists
1912026-07-31T16:46:20.909ZUsing 10.151.2.100 as gateway ip
1922026-07-31T16:46:20.909Zusing 18:C0:4D:0D:A0:2A as gateway mac
1932026-07-31T16:46:20.909Zconfiguring SoftNPU ARP entry
1942026-07-31T16:46:20.937Zconfiguring SoftNPU proxy ARP
1952026-07-31T16:46:20.992ZSoftNPU state:
1962026-07-31T16:46:20.992Z local v6:
1972026-07-31T16:46:20.992Z local v4:
1982026-07-31T16:46:20.992Z router v6_idx:
1992026-07-31T16:46:20.992Z router v6_routes:
2002026-07-31T16:46:20.992Z router v4_idx:
2012026-07-31T16:46:20.992Z router v4_routes:
2022026-07-31T16:46:20.992Z resolver v4:
2032026-07-31T16:46:20.992Z 10.151.2.100 -> 18:c0:4d:0d:a0:2a
2042026-07-31T16:46:20.992Z resolver v6:
2052026-07-31T16:46:20.992Z nat v4:
2062026-07-31T16:46:20.993Z nat_v6:
2072026-07-31T16:46:20.993Z attached subnet v4:
2082026-07-31T16:46:20.993Z attached subnet v6:
2092026-07-31T16:46:20.993Z port_mac:
2102026-07-31T16:46:20.993Z proxy arp:
2112026-07-31T16:46:20.993Z 10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
2122026-07-31T16:46:20.993Zcreated virtual hardware
2132026-07-31T16:46:21.017Z
2142026-07-31T16:46:21.017Zreal 15.195265433
2152026-07-31T16:46:21.017Zuser 0.590783280
2162026-07-31T16:46:21.017Zsys 3.321968464
2172026-07-31T16:46:21.017Ztrap 0.000069249
2182026-07-31T16:46:21.017Ztflt 0.000171566
2192026-07-31T16:46:21.017Zdflt 0.000496145
2202026-07-31T16:46:21.017Zkflt 0.000005220
2212026-07-31T16:46:21.017Zlock 11:50.245809613
2222026-07-31T16:46:21.017Zslp 45.944700129
2232026-07-31T16:46:21.017Zlat 0.176746796
2242026-07-31T16:46:21.017Zstop 0.277759081
2252026-07-31T16:46:21.017Z+ tar xf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
2262026-07-31T16:46:21.044Z+ sed -E -i~ 's/(m2|u2)(.*\.vdev)/\/scratch\/\1\2/g' pkg/config.toml
2272026-07-31T16:46:21.089Z+ diff -u pkg/config.toml~ pkg/config.toml
2282026-07-31T16:46:21.089Z--- pkg/config.toml~ Mon Jul 24 01:21:28 2006
2292026-07-31T16:46:21.089Z+++ pkg/config.toml Fri Jul 31 16:46:21 2026
2302026-07-31T16:46:21.089Z@@ -93,18 +93,18 @@
2312026-07-31T16:46:21.090Z [external_disks]
2322026-07-31T16:46:21.090Z kind = "hardcoded"
2332026-07-31T16:46:21.090Z vdevs = [
2342026-07-31T16:46:21.090Z- "m2_0.vdev",
2352026-07-31T16:46:21.090Z- "m2_1.vdev",
2362026-07-31T16:46:21.090Z+ "/scratch/m2_0.vdev",
2372026-07-31T16:46:21.090Z+ "/scratch/m2_1.vdev",
2382026-07-31T16:46:21.090Z
2392026-07-31T16:46:21.090Z- "u2_0.vdev",
2402026-07-31T16:46:21.090Z- "u2_1.vdev",
2412026-07-31T16:46:21.090Z- "u2_2.vdev",
2422026-07-31T16:46:21.090Z- "u2_3.vdev",
2432026-07-31T16:46:21.090Z- "u2_4.vdev",
2442026-07-31T16:46:21.090Z- "u2_5.vdev",
2452026-07-31T16:46:21.090Z- "u2_6.vdev",
2462026-07-31T16:46:21.090Z- "u2_7.vdev",
2472026-07-31T16:46:21.090Z- "u2_8.vdev",
2482026-07-31T16:46:21.090Z+ "/scratch/u2_0.vdev",
2492026-07-31T16:46:21.090Z+ "/scratch/u2_1.vdev",
2502026-07-31T16:46:21.090Z+ "/scratch/u2_2.vdev",
2512026-07-31T16:46:21.090Z+ "/scratch/u2_3.vdev",
2522026-07-31T16:46:21.090Z+ "/scratch/u2_4.vdev",
2532026-07-31T16:46:21.090Z+ "/scratch/u2_5.vdev",
2542026-07-31T16:46:21.090Z+ "/scratch/u2_6.vdev",
2552026-07-31T16:46:21.090Z+ "/scratch/u2_7.vdev",
2562026-07-31T16:46:21.090Z+ "/scratch/u2_8.vdev",
2572026-07-31T16:46:21.090Z ]
2582026-07-31T16:46:21.090Z disks = []
2592026-07-31T16:46:21.090Z
2602026-07-31T16:46:21.090Z+ true
2612026-07-31T16:46:21.090Z++ grep -c -E 'u2.*\.vdev' pkg/config.toml
2622026-07-31T16:46:21.090Z+ EXPECTED_ZPOOL_COUNT=9
2632026-07-31T16:46:21.090Z+ echo 'expected number of zpools is 9'
2642026-07-31T16:46:21.090Zexpected number of zpools is 9
2652026-07-31T16:46:21.090Z++ sed -n 's/silo_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2662026-07-31T16:46:21.090Z+ SILO_NAME=recovery
2672026-07-31T16:46:21.090Z++ sed -n 's/external_dns_zone_name = "\(.*\)"/\1/p' pkg/config-rss.toml
2682026-07-31T16:46:21.091Z+ EXTERNAL_DNS_DOMAIN=oxide.test
2692026-07-31T16:46:21.091Z+ sed -i~ '
2702026-07-31T16:46:21.091Z /^external_dns_ips/c\
2712026-07-31T16:46:21.091Zexternal_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2722026-07-31T16:46:21.091Z /^\[\[service_ip_pools/,/^$/ {
2732026-07-31T16:46:21.091Z /^ranges =/c\
2742026-07-31T16:46:21.091Zranges = \[ { first = "10.151.2.164", last = "10.151.2.173" } \]
2752026-07-31T16:46:21.091Z }
2762026-07-31T16:46:21.091Z /^infra_ip_first/c\
2772026-07-31T16:46:21.091Zinfra_ip_first = "10.151.2.174"
2782026-07-31T16:46:21.091Z /^infra_ip_last/c\
2792026-07-31T16:46:21.091Zinfra_ip_last = "10.151.2.174"
2802026-07-31T16:46:21.091Z /^\[\[rack_network_config.ports/,/^$/ {
2812026-07-31T16:46:21.091Z /^routes/c\
2822026-07-31T16:46:21.091Zroutes = \[{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}\]
2832026-07-31T16:46:21.091Z /^addresses/c\
2842026-07-31T16:46:21.091Zaddresses = \[{address = {type = "static", ip_net = "10.151.2.174/24"} }\]
2852026-07-31T16:46:21.091Z }
2862026-07-31T16:46:21.091Z' pkg/config-rss.toml
2872026-07-31T16:46:21.091Z+ diff -u pkg/config-rss.toml~ pkg/config-rss.toml
2882026-07-31T16:46:21.091Z--- pkg/config-rss.toml~ Mon Jul 24 01:21:28 2006
2892026-07-31T16:46:21.091Z+++ pkg/config-rss.toml Fri Jul 31 16:46:21 2026
2902026-07-31T16:46:21.091Z@@ -33,7 +33,7 @@
2912026-07-31T16:46:21.091Z # the DNS domain delegated to the rack by the customer. Each of these addresses
2922026-07-31T16:46:21.091Z # must be contained in one of the "internal services" IP Pool ranges listed
2932026-07-31T16:46:21.091Z # below.
2942026-07-31T16:46:21.091Z-external_dns_ips = [ "192.168.1.20", "192.168.1.21" ]
2952026-07-31T16:46:21.091Z+external_dns_ips = [ "10.151.2.164", "10.151.2.165" ]
2962026-07-31T16:46:21.091Z
2972026-07-31T16:46:21.091Z # Initial TLS certificates for the external API
2982026-07-31T16:46:21.091Z #
2992026-07-31T16:46:21.091Z@@ -71,7 +71,7 @@
3002026-07-31T16:46:21.091Z [[service_ip_pools]]
3012026-07-31T16:46:21.091Z name = "oxide-service-pool-v4"
3022026-07-31T16:46:21.091Z description = "IPv4 IP Pool for Oxide Services"
3032026-07-31T16:46:21.091Z-ranges = [ { first = "192.168.1.20", last = "192.168.1.29" } ]
3042026-07-31T16:46:21.091Z+ranges = [ { first = "10.151.2.164", last = "10.151.2.173" } ]
3052026-07-31T16:46:21.091Z
3062026-07-31T16:46:21.091Z # TODO - this configuration is subject to change going forward. Ultimately these
3072026-07-31T16:46:21.091Z # parameters should be provided to the control plane via wicket, but we need to
3082026-07-31T16:46:21.092Z@@ -92,8 +92,8 @@
3092026-07-31T16:46:21.092Z # A range of IP addresses used by Boundary Services on the external network. In
3102026-07-31T16:46:21.092Z # a real system, these would be addresses of the uplink ports on the Sidecar.
3112026-07-31T16:46:21.092Z # With softnpu, only one address is used.
3122026-07-31T16:46:21.092Z-infra_ip_first = "192.168.1.30"
3132026-07-31T16:46:21.092Z-infra_ip_last = "192.168.1.30"
3142026-07-31T16:46:21.092Z+infra_ip_first = "10.151.2.174"
3152026-07-31T16:46:21.092Z+infra_ip_last = "10.151.2.174"
3162026-07-31T16:46:21.092Z
3172026-07-31T16:46:21.092Z # Configurations for BGP routers to run on the scrimlets.
3182026-07-31T16:46:21.092Z bgp = []
3192026-07-31T16:46:21.092Z@@ -101,9 +101,9 @@
3202026-07-31T16:46:21.092Z # You can configure multiple uplinks by repeating the following stanza
3212026-07-31T16:46:21.092Z [[rack_network_config.ports]]
3222026-07-31T16:46:21.092Z # Routes associated with this port.
3232026-07-31T16:46:21.092Z-routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}]
3242026-07-31T16:46:21.092Z+routes = [{nexthop = "10.151.2.100", destination = "0.0.0.0/0"}]
3252026-07-31T16:46:21.092Z # Addresses associated with this port.
3262026-07-31T16:46:21.092Z-addresses = [{address = {type = "static", ip_net = "192.168.1.30/24"}}]
3272026-07-31T16:46:21.117Z+addresses = [{address = {type = "static", ip_net = "10.151.2.174/24"} }]
3282026-07-31T16:46:21.117Z # Name of the uplink port. This should always be "qsfp0" when using softnpu.
3292026-07-31T16:46:21.117Z port = "qsfp0"
3302026-07-31T16:46:21.117Z # The speed of this port.
3312026-07-31T16:46:21.117Z+ true
3322026-07-31T16:46:21.117Z+ tar rvf out/omicron-sled-agent.tar pkg/config-rss.toml pkg/config.toml
3332026-07-31T16:46:21.117Z+ rm -f pkg/config-rss.toml pkg/config-rss.toml~ pkg/config.toml pkg/config.toml~
3342026-07-31T16:46:21.117Za pkg/config-rss.toml 8K
3352026-07-31T16:46:21.117Za pkg/config.toml 6K
3362026-07-31T16:46:21.117Z+ TLS_NAME=recovery.sys.oxide.test
3372026-07-31T16:46:21.117Z+ openssl req -newkey rsa:4096 -x509 -sha256 -days 3 -nodes -out pkg/initial-tls-cert.pem -keyout pkg/initial-tls-key.pem -subj /CN=recovery.sys.oxide.test -addext subjectAltName=DNS:recovery.sys.oxide.test -addext basicConstraints=critical,CA:FALSE -config /dev/stdin
3382026-07-31T16:46:21.117Z.+..+....+......+...+..+...+...+...+.+...+........+............+.......+.....+......+....+++++++++++++++++++++++++++++++++++++++++++++*............+..+.+..+...+.............+..+.......+......+......+...+..+...+.......+.....+...+.........................+......+++++++++++++++++++++++++++++++++++++++++++++*.........+............+.+......+.........+.....+.+..................+..+.+...............+...+..+............+++++
3392026-07-31T16:46:21.151Z.+...+.....+.+........+......+.......+...+..+.......+.....+.+..+....+...+........+...+++++++++++++++++++++++++++++++++++++++++++++*......+...........+......+...+...+.............+..+...+................+++++++++++++++++++++++++++++++++++++++++++++*..+......+..........+...........+.............+.....+.+...............+.........+..+....+.....+..........+++++
3402026-07-31T16:46:21.176Z-----
3412026-07-31T16:46:21.200Z+ tar rvf out/omicron-sled-agent.tar pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3422026-07-31T16:46:21.200Za pkg/initial-tls-cert.pem 2K
3432026-07-31T16:46:21.200Za pkg/initial-tls-key.pem 4K
3442026-07-31T16:46:21.200Z+ rm -f pkg/initial-tls-cert.pem pkg/initial-tls-key.pem
3452026-07-31T16:46:21.200Z+ rmdir pkg
3462026-07-31T16:46:21.200Z+ E2E_TLS_CERT=/opt/oxide/sled-agent/pkg/initial-tls-cert.pem
3472026-07-31T16:46:21.200Z+ pfexec mkdir -p /usr/oxide
3482026-07-31T16:46:21.200Z+ pfexec curl -sSfL -o /usr/oxide/oxide http://catacomb.eng.oxide.computer:12346/oxide-v2025112000
3492026-07-31T16:46:21.791Z+ pfexec chmod +x /usr/oxide/oxide
3502026-07-31T16:46:21.815Z+ curl -sSfL -o debian-11-genericcloud-amd64.raw http://catacomb.eng.oxide.computer:12346/debian-11-genericcloud-amd64.raw
3512026-07-31T16:46:40.418Z+ OMICRON_NO_UNINSTALL=1
3522026-07-31T16:46:40.418Z+ ptime -m pfexec ./target/release/omicron-package -t test install
3532026-07-31T16:46:40.442ZLogging to: /opt/oxide/work/out/LOG
3542026-07-31T16:46:41.468Z
3552026-07-31T16:46:41.468Zreal 1.043202109
3562026-07-31T16:46:41.468Zuser 0.193340163
3572026-07-31T16:46:41.493Zsys 7.024482180
3582026-07-31T16:46:41.493Ztrap 0.000008868
3592026-07-31T16:46:41.493Ztflt 0.000403886
3602026-07-31T16:46:41.493Zdflt 0.000197193
3612026-07-31T16:46:41.493Zkflt 0.000000000
3622026-07-31T16:46:41.493Zlock 1:33.689883650
3632026-07-31T16:46:41.493Zslp 1.223289932
3642026-07-31T16:46:41.493Zlat 0.041857694
3652026-07-31T16:46:41.493Zstop 0.063399743
3662026-07-31T16:46:41.493Z+ retry=0
3672026-07-31T16:46:41.493Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3682026-07-31T16:46:41.493Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3692026-07-31T16:46:41.493Z+ [[ 0 -gt 30 ]]
3702026-07-31T16:46:41.493Z+ sleep 1
3712026-07-31T16:46:42.485Z+ retry=1
3722026-07-31T16:46:42.485Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3732026-07-31T16:46:42.509Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3742026-07-31T16:46:42.509Z+ [[ 1 -gt 30 ]]
3752026-07-31T16:46:42.509Z+ sleep 1
3762026-07-31T16:46:43.499Z+ retry=2
3772026-07-31T16:46:43.499Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3782026-07-31T16:46:43.523Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 0 ms: Could not connect to server
3792026-07-31T16:46:43.523Z+ [[ 2 -gt 30 ]]
3802026-07-31T16:46:43.523Z+ sleep 1
3812026-07-31T16:46:44.519Z+ retry=3
3822026-07-31T16:46:44.520Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3832026-07-31T16:46:45.537Zcurl: (28) Connection timed out after 1000 milliseconds
3842026-07-31T16:46:45.561Z+ [[ 3 -gt 30 ]]
3852026-07-31T16:46:45.561Z+ sleep 1
3862026-07-31T16:46:46.548Z+ retry=4
3872026-07-31T16:46:46.548Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3882026-07-31T16:46:47.561Zcurl: (28) Connection timed out after 1000 milliseconds
3892026-07-31T16:46:47.585Z+ [[ 4 -gt 30 ]]
3902026-07-31T16:46:47.585Z+ sleep 1
3912026-07-31T16:46:48.566Z+ retry=5
3922026-07-31T16:46:48.566Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3932026-07-31T16:46:49.579Zcurl: (28) Connection timed out after 1000 milliseconds
3942026-07-31T16:46:49.603Z+ [[ 5 -gt 30 ]]
3952026-07-31T16:46:49.603Z+ sleep 1
3962026-07-31T16:46:50.585Z+ retry=6
3972026-07-31T16:46:50.585Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
3982026-07-31T16:46:51.601Zcurl: (28) Connection timed out after 1000 milliseconds
3992026-07-31T16:46:51.625Z+ [[ 6 -gt 30 ]]
4002026-07-31T16:46:51.625Z+ sleep 1
4012026-07-31T16:46:52.607Z+ retry=7
4022026-07-31T16:46:52.607Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4032026-07-31T16:46:53.626Zcurl: (28) Connection timed out after 1000 milliseconds
4042026-07-31T16:46:53.651Z+ [[ 7 -gt 30 ]]
4052026-07-31T16:46:53.651Z+ sleep 1
4062026-07-31T16:46:54.632Z+ retry=8
4072026-07-31T16:46:54.632Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4082026-07-31T16:46:55.650Zcurl: (28) Connection timed out after 1000 milliseconds
4092026-07-31T16:46:55.675Z+ [[ 8 -gt 30 ]]
4102026-07-31T16:46:55.675Z+ sleep 1
4112026-07-31T16:46:56.658Z+ retry=9
4122026-07-31T16:46:56.658Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4132026-07-31T16:46:57.669Zcurl: (28) Connection timed out after 1000 milliseconds
4142026-07-31T16:46:57.693Z+ [[ 9 -gt 30 ]]
4152026-07-31T16:46:57.693Z+ sleep 1
4162026-07-31T16:46:58.684Z+ retry=10
4172026-07-31T16:46:58.684Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4182026-07-31T16:46:59.696Zcurl: (28) Connection timed out after 1000 milliseconds
4192026-07-31T16:46:59.720Z+ [[ 10 -gt 30 ]]
4202026-07-31T16:46:59.720Z+ sleep 1
4212026-07-31T16:47:00.701Z+ retry=11
4222026-07-31T16:47:00.731Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4232026-07-31T16:47:01.832Zcurl: (28) Connection timed out after 1000 milliseconds
4242026-07-31T16:47:01.856Z+ [[ 11 -gt 30 ]]
4252026-07-31T16:47:01.856Z+ sleep 1
4262026-07-31T16:47:02.838Z+ retry=12
4272026-07-31T16:47:02.838Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4282026-07-31T16:47:03.851Zcurl: (28) Connection timed out after 1000 milliseconds
4292026-07-31T16:47:03.875Z+ [[ 12 -gt 30 ]]
4302026-07-31T16:47:03.875Z+ sleep 1
4312026-07-31T16:47:04.857Z+ retry=13
4322026-07-31T16:47:04.857Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4332026-07-31T16:47:05.758Zcurl: (7) Failed to connect to fd00:1122:3344:101::2 port 12224 after 889 ms: Could not connect to server
4342026-07-31T16:47:05.782Z+ [[ 13 -gt 30 ]]
4352026-07-31T16:47:05.782Z+ sleep 1
4362026-07-31T16:47:06.764Z+ retry=14
4372026-07-31T16:47:06.764Z+ curl --max-time 1 --head --silent --show-error -o /dev/null 'http://[fd00:1122:3344:101::2]:12224/'
4382026-07-31T16:47:06.788Z+ pfexec zlogin sidecar_softnpu /softnpu/scadm --server /softnpu/server --client /softnpu/client standalone dump-state
4392026-07-31T16:47:06.813Zlocal v6:
4402026-07-31T16:47:06.813Zfe80::24b1:ccff:fefc:9103
4412026-07-31T16:47:06.813Zfd56:7d8d:ac7c:1b6e::1
4422026-07-31T16:47:06.813Zlocal v4:
4432026-07-31T16:47:06.813Z10.151.2.174
4442026-07-31T16:47:06.813Zrouter v6_idx:
4452026-07-31T16:47:06.813Zrouter v6_routes:
4462026-07-31T16:47:06.813Zrouter v4_idx:
4472026-07-31T16:47:06.813Zrouter v4_routes:
4482026-07-31T16:47:06.813Zresolver v4:
4492026-07-31T16:47:06.813Z10.151.2.100 -> 18:c0:4d:0d:a0:2a
4502026-07-31T16:47:06.813Zresolver v6:
4512026-07-31T16:47:06.813Zfe80::74b1:75ff:fec3:780b -> 76:b1:75:c3:78:0b
4522026-07-31T16:47:06.813Zfe80::24b1:ccff:fefc:9103 -> 26:b1:cc:fc:91:03
4532026-07-31T16:47:06.813Znat v4:
4542026-07-31T16:47:06.813Znat_v6:
4552026-07-31T16:47:06.813Zattached subnet v4:
4562026-07-31T16:47:06.813Zattached subnet v6:
4572026-07-31T16:47:06.813Zport_mac:
4582026-07-31T16:47:06.813Z1: a8:40:25:25:77:dd
4592026-07-31T16:47:06.814Z2: a8:40:25:25:77:fd
4602026-07-31T16:47:06.814Zproxy arp:
4612026-07-31T16:47:06.814Z10.151.2.164/10.151.2.227: a8:e1:de:01:70:1d
4622026-07-31T16:47:06.838Z+ retry=0
4632026-07-31T16:47:06.838Z+++ zoneadm list -n
4642026-07-31T16:47:06.838Z+++ grep oxz_ntp
4652026-07-31T16:47:06.838Z++ pfexec svcs -z -Hostate oxide/ntp
4662026-07-31T16:47:06.838Zsvcs: zone '-Hostate': entity not found
4672026-07-31T16:47:06.838Z++ true
4682026-07-31T16:47:06.839Z+ [[ '' != online ]]
4692026-07-31T16:47:06.839Z+ [[ 0 -gt 60 ]]
4702026-07-31T16:47:06.839Z+ sleep 1
4712026-07-31T16:47:07.837Z+ retry=1
4722026-07-31T16:47:07.862Z+++ zoneadm list -n
4732026-07-31T16:47:07.862Z+++ grep oxz_ntp
4742026-07-31T16:47:07.862Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
4752026-07-31T16:47:07.862Zsvcs: Could not bind to repository server for zone oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676: repository server unavailable
4762026-07-31T16:47:07.862Z++ true
4772026-07-31T16:47:07.862Z+ [[ '' != online ]]
4782026-07-31T16:47:07.862Z+ [[ 1 -gt 60 ]]
4792026-07-31T16:47:07.862Z+ sleep 1
4802026-07-31T16:47:08.863Z+ retry=2
4812026-07-31T16:47:08.887Z+++ zoneadm list -n
4822026-07-31T16:47:08.887Z+++ grep oxz_ntp
4832026-07-31T16:47:08.888Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
4842026-07-31T16:47:08.912Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4852026-07-31T16:47:08.912Z++ true
4862026-07-31T16:47:08.912Z+ [[ '' != online ]]
4872026-07-31T16:47:08.912Z+ [[ 2 -gt 60 ]]
4882026-07-31T16:47:08.912Z+ sleep 1
4892026-07-31T16:47:09.896Z+ retry=3
4902026-07-31T16:47:09.920Z+++ zoneadm list -n
4912026-07-31T16:47:09.920Z+++ grep oxz_ntp
4922026-07-31T16:47:09.920Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
4932026-07-31T16:47:09.944Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
4942026-07-31T16:47:09.945Z++ true
4952026-07-31T16:47:09.945Z+ [[ '' != online ]]
4962026-07-31T16:47:09.945Z+ [[ 3 -gt 60 ]]
4972026-07-31T16:47:09.945Z+ sleep 1
4982026-07-31T16:47:10.927Z+ retry=4
4992026-07-31T16:47:10.951Z+++ zoneadm list -n
5002026-07-31T16:47:10.952Z+++ grep oxz_ntp
5012026-07-31T16:47:10.952Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5022026-07-31T16:47:10.975Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
5032026-07-31T16:47:10.975Z++ true
5042026-07-31T16:47:10.975Z+ [[ '' != online ]]
5052026-07-31T16:47:10.975Z+ [[ 4 -gt 60 ]]
5062026-07-31T16:47:10.976Z+ sleep 1
5072026-07-31T16:47:11.957Z+ retry=5
5082026-07-31T16:47:11.981Z+++ zoneadm list -n
5092026-07-31T16:47:11.982Z+++ grep oxz_ntp
5102026-07-31T16:47:11.982Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5112026-07-31T16:47:12.006Zsvcs: Pattern 'oxide/ntp' doesn't match any instances
5122026-07-31T16:47:12.006Z++ true
5132026-07-31T16:47:12.006Z+ [[ '' != online ]]
5142026-07-31T16:47:12.006Z+ [[ 5 -gt 60 ]]
5152026-07-31T16:47:12.006Z+ sleep 1
5162026-07-31T16:47:12.992Z+ retry=6
5172026-07-31T16:47:13.017Z+++ zoneadm list -n
5182026-07-31T16:47:13.017Z+++ grep oxz_ntp
5192026-07-31T16:47:13.017Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5202026-07-31T16:47:13.017Z+ [[ offline != online ]]
5212026-07-31T16:47:13.017Z+ [[ 6 -gt 60 ]]
5222026-07-31T16:47:13.017Z+ sleep 1
5232026-07-31T16:47:14.016Z+ retry=7
5242026-07-31T16:47:14.041Z+++ zoneadm list -n
5252026-07-31T16:47:14.041Z+++ grep oxz_ntp
5262026-07-31T16:47:14.041Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5272026-07-31T16:47:14.041Z+ [[ offline != online ]]
5282026-07-31T16:47:14.041Z+ [[ 7 -gt 60 ]]
5292026-07-31T16:47:14.041Z+ sleep 1
5302026-07-31T16:47:15.039Z+ retry=8
5312026-07-31T16:47:15.064Z+++ zoneadm list -n
5322026-07-31T16:47:15.064Z+++ grep oxz_ntp
5332026-07-31T16:47:15.064Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5342026-07-31T16:47:15.064Z+ [[ offline != online ]]
5352026-07-31T16:47:15.064Z+ [[ 8 -gt 60 ]]
5362026-07-31T16:47:15.064Z+ sleep 1
5372026-07-31T16:47:16.062Z+ retry=9
5382026-07-31T16:47:16.086Z+++ zoneadm list -n
5392026-07-31T16:47:16.086Z+++ grep oxz_ntp
5402026-07-31T16:47:16.086Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5412026-07-31T16:47:16.086Z+ [[ offline != online ]]
5422026-07-31T16:47:16.086Z+ [[ 9 -gt 60 ]]
5432026-07-31T16:47:16.086Z+ sleep 1
5442026-07-31T16:47:17.085Z+ retry=10
5452026-07-31T16:47:17.110Z+++ zoneadm list -n
5462026-07-31T16:47:17.110Z+++ grep oxz_ntp
5472026-07-31T16:47:17.110Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5482026-07-31T16:47:17.110Z+ [[ offline != online ]]
5492026-07-31T16:47:17.110Z+ [[ 10 -gt 60 ]]
5502026-07-31T16:47:17.110Z+ sleep 1
5512026-07-31T16:47:18.109Z+ retry=11
5522026-07-31T16:47:18.134Z+++ zoneadm list -n
5532026-07-31T16:47:18.134Z+++ grep oxz_ntp
5542026-07-31T16:47:18.134Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5552026-07-31T16:47:18.134Z+ [[ offline != online ]]
5562026-07-31T16:47:18.134Z+ [[ 11 -gt 60 ]]
5572026-07-31T16:47:18.134Z+ sleep 1
5582026-07-31T16:47:19.133Z+ retry=12
5592026-07-31T16:47:19.158Z+++ zoneadm list -n
5602026-07-31T16:47:19.158Z+++ grep oxz_ntp
5612026-07-31T16:47:19.158Z++ pfexec svcs -z oxz_ntp_96109005-bcdd-44be-9f94-c189894a9676 -Hostate oxide/ntp
5622026-07-31T16:47:19.158Z+ [[ online != online ]]
5632026-07-31T16:47:19.158Z+ echo 'Waited for chrony: 12s'
5642026-07-31T16:47:19.158Z+ retry=0
5652026-07-31T16:47:19.158ZWaited for chrony: 12s
5662026-07-31T16:47:19.158Z+ zoneadm list
5672026-07-31T16:47:19.158Z+ grep nexus
5682026-07-31T16:47:19.183Z+ [[ 0 -gt 300 ]]
5692026-07-31T16:47:19.183Z+ sleep 1
5702026-07-31T16:47:20.165Z+ retry=1
5712026-07-31T16:47:20.189Z+ zoneadm list
5722026-07-31T16:47:20.189Z+ grep nexus
5732026-07-31T16:47:20.189Z+ [[ 1 -gt 300 ]]
5742026-07-31T16:47:20.189Z+ sleep 1
5752026-07-31T16:47:21.177Z+ retry=2
5762026-07-31T16:47:21.201Z+ zoneadm list
5772026-07-31T16:47:21.201Z+ grep nexus
5782026-07-31T16:47:21.201Z+ [[ 2 -gt 300 ]]
5792026-07-31T16:47:21.201Z+ sleep 1
5802026-07-31T16:47:22.190Z+ retry=3
5812026-07-31T16:47:22.214Z+ zoneadm list
5822026-07-31T16:47:22.214Z+ grep nexus
5832026-07-31T16:47:22.214Z+ [[ 3 -gt 300 ]]
5842026-07-31T16:47:22.214Z+ sleep 1
5852026-07-31T16:47:23.203Z+ retry=4
5862026-07-31T16:47:23.227Z+ zoneadm list
5872026-07-31T16:47:23.228Z+ grep nexus
5882026-07-31T16:47:23.228Z+ [[ 4 -gt 300 ]]
5892026-07-31T16:47:23.228Z+ sleep 1
5902026-07-31T16:47:24.213Z+ retry=5
5912026-07-31T16:47:24.237Z+ zoneadm list
5922026-07-31T16:47:24.237Z+ grep nexus
5932026-07-31T16:47:24.237Z+ [[ 5 -gt 300 ]]
5942026-07-31T16:47:24.237Z+ sleep 1
5952026-07-31T16:47:25.230Z+ retry=6
5962026-07-31T16:47:25.254Z+ zoneadm list
5972026-07-31T16:47:25.254Z+ grep nexus
5982026-07-31T16:47:25.254Z+ [[ 6 -gt 300 ]]
5992026-07-31T16:47:25.254Z+ sleep 1
6002026-07-31T16:47:26.244Z+ retry=7
6012026-07-31T16:47:26.268Z+ zoneadm list
6022026-07-31T16:47:26.268Z+ grep nexus
6032026-07-31T16:47:26.268Z+ [[ 7 -gt 300 ]]
6042026-07-31T16:47:26.268Z+ sleep 1
6052026-07-31T16:47:27.326Z+ retry=8
6062026-07-31T16:47:27.351Z+ grep nexus
6072026-07-31T16:47:27.525Z+ zoneadm list
6082026-07-31T16:47:27.570Z+ [[ 8 -gt 300 ]]
6092026-07-31T16:47:27.570Z+ sleep 1
6102026-07-31T16:47:28.577Z+ retry=9
6112026-07-31T16:47:28.601Z+ zoneadm list
6122026-07-31T16:47:28.601Z+ grep nexus
6132026-07-31T16:47:28.626Z+ [[ 9 -gt 300 ]]
6142026-07-31T16:47:28.627Z+ sleep 1
6152026-07-31T16:47:29.631Z+ retry=10
6162026-07-31T16:47:29.656Z+ zoneadm list
6172026-07-31T16:47:29.656Z+ grep nexus
6182026-07-31T16:47:29.656Z+ [[ 10 -gt 300 ]]
6192026-07-31T16:47:29.656Z+ sleep 1
6202026-07-31T16:47:30.670Z+ retry=11
6212026-07-31T16:47:30.694Z+ zoneadm list
6222026-07-31T16:47:30.694Z+ grep nexus
6232026-07-31T16:47:30.694Z+ [[ 11 -gt 300 ]]
6242026-07-31T16:47:30.694Z+ sleep 1
6252026-07-31T16:47:31.684Z+ retry=12
6262026-07-31T16:47:31.708Z+ zoneadm list
6272026-07-31T16:47:31.709Z+ grep nexus
6282026-07-31T16:47:31.709Z+ [[ 12 -gt 300 ]]
6292026-07-31T16:47:31.709Z+ sleep 1
6302026-07-31T16:47:32.707Z+ retry=13
6312026-07-31T16:47:32.731Z+ zoneadm list
6322026-07-31T16:47:32.732Z+ grep nexus
6332026-07-31T16:47:32.732Z+ [[ 13 -gt 300 ]]
6342026-07-31T16:47:32.732Z+ sleep 1
6352026-07-31T16:47:33.722Z+ retry=14
6362026-07-31T16:47:33.746Z+ zoneadm list
6372026-07-31T16:47:33.746Z+ grep nexus
6382026-07-31T16:47:33.746Z+ [[ 14 -gt 300 ]]
6392026-07-31T16:47:33.746Z+ sleep 1
6402026-07-31T16:47:34.734Z+ retry=15
6412026-07-31T16:47:34.758Z+ zoneadm list
6422026-07-31T16:47:34.758Z+ grep nexus
6432026-07-31T16:47:34.758Z+ [[ 15 -gt 300 ]]
6442026-07-31T16:47:34.758Z+ sleep 1
6452026-07-31T16:47:35.747Z+ retry=16
6462026-07-31T16:47:35.771Z+ zoneadm list
6472026-07-31T16:47:35.771Z+ grep nexus
6482026-07-31T16:47:35.771Z+ [[ 16 -gt 300 ]]
6492026-07-31T16:47:35.771Z+ sleep 1
6502026-07-31T16:47:36.762Z+ retry=17
6512026-07-31T16:47:36.786Z+ zoneadm list
6522026-07-31T16:47:36.786Z+ grep nexus
6532026-07-31T16:47:36.786Z+ [[ 17 -gt 300 ]]
6542026-07-31T16:47:36.786Z+ sleep 1
6552026-07-31T16:47:37.774Z+ retry=18
6562026-07-31T16:47:37.798Z+ zoneadm list
6572026-07-31T16:47:37.798Z+ grep nexus
6582026-07-31T16:47:37.798Z+ [[ 18 -gt 300 ]]
6592026-07-31T16:47:37.798Z+ sleep 1
6602026-07-31T16:47:38.787Z+ retry=19
6612026-07-31T16:47:38.811Z+ zoneadm list
6622026-07-31T16:47:38.812Z+ grep nexus
6632026-07-31T16:47:38.812Z+ [[ 19 -gt 300 ]]
6642026-07-31T16:47:38.812Z+ sleep 1
6652026-07-31T16:47:39.799Z+ retry=20
6662026-07-31T16:47:39.824Z+ zoneadm list
6672026-07-31T16:47:39.824Z+ grep nexus
6682026-07-31T16:47:39.824Z+ [[ 20 -gt 300 ]]
6692026-07-31T16:47:39.824Z+ sleep 1
6702026-07-31T16:47:40.813Z+ retry=21
6712026-07-31T16:47:40.838Z+ zoneadm list
6722026-07-31T16:47:40.838Z+ grep nexus
6732026-07-31T16:47:40.838Z+ [[ 21 -gt 300 ]]
6742026-07-31T16:47:40.838Z+ sleep 1
6752026-07-31T16:47:41.827Z+ retry=22
6762026-07-31T16:47:41.851Z+ zoneadm list
6772026-07-31T16:47:41.851Z+ grep nexus
6782026-07-31T16:47:41.851Z+ [[ 22 -gt 300 ]]
6792026-07-31T16:47:41.851Z+ sleep 1
6802026-07-31T16:47:42.840Z+ retry=23
6812026-07-31T16:47:42.864Z+ zoneadm list
6822026-07-31T16:47:42.864Z+ grep nexus
6832026-07-31T16:47:42.864Z+ [[ 23 -gt 300 ]]
6842026-07-31T16:47:42.864Z+ sleep 1
6852026-07-31T16:47:43.853Z+ retry=24
6862026-07-31T16:47:43.879Z+ zoneadm list
6872026-07-31T16:47:43.879Z+ grep nexus
6882026-07-31T16:47:43.879Z+ [[ 24 -gt 300 ]]
6892026-07-31T16:47:43.879Z+ sleep 1
6902026-07-31T16:47:44.866Z+ retry=25
6912026-07-31T16:47:44.890Z+ zoneadm list
6922026-07-31T16:47:44.890Z+ grep nexus
6932026-07-31T16:47:44.890Z+ [[ 25 -gt 300 ]]
6942026-07-31T16:47:44.890Z+ sleep 1
6952026-07-31T16:47:45.879Z+ retry=26
6962026-07-31T16:47:45.903Z+ zoneadm list
6972026-07-31T16:47:45.903Z+ grep nexus
6982026-07-31T16:47:45.903Z+ [[ 26 -gt 300 ]]
6992026-07-31T16:47:45.903Z+ sleep 1
7002026-07-31T16:47:46.894Z+ retry=27
7012026-07-31T16:47:46.918Z+ zoneadm list
7022026-07-31T16:47:46.918Z+ grep nexus
7032026-07-31T16:47:46.918Z+ [[ 27 -gt 300 ]]
7042026-07-31T16:47:46.918Z+ sleep 1
7052026-07-31T16:47:47.908Z+ retry=28
7062026-07-31T16:47:47.932Z+ zoneadm list
7072026-07-31T16:47:47.932Z+ grep nexus
7082026-07-31T16:47:47.932Z+ [[ 28 -gt 300 ]]
7092026-07-31T16:47:47.932Z+ sleep 1
7102026-07-31T16:47:48.930Z+ retry=29
7112026-07-31T16:47:48.955Z+ zoneadm list
7122026-07-31T16:47:48.955Z+ grep nexus
7132026-07-31T16:47:48.955Z+ [[ 29 -gt 300 ]]
7142026-07-31T16:47:48.955Z+ sleep 1
7152026-07-31T16:47:49.947Z+ retry=30
7162026-07-31T16:47:49.971Z+ zoneadm list
7172026-07-31T16:47:49.971Z+ grep nexus
7182026-07-31T16:47:49.971Z+ [[ 30 -gt 300 ]]
7192026-07-31T16:47:49.971Z+ sleep 1
7202026-07-31T16:47:50.963Z+ retry=31
7212026-07-31T16:47:50.987Z+ zoneadm list
7222026-07-31T16:47:50.987Z+ grep nexus
7232026-07-31T16:47:50.987Z+ [[ 31 -gt 300 ]]
7242026-07-31T16:47:50.987Z+ sleep 1
7252026-07-31T16:47:51.979Z+ retry=32
7262026-07-31T16:47:52.004Z+ zoneadm list
7272026-07-31T16:47:52.004Z+ grep nexus
7282026-07-31T16:47:52.004Z+ [[ 32 -gt 300 ]]
7292026-07-31T16:47:52.004Z+ sleep 1
7302026-07-31T16:47:52.993Z+ retry=33
7312026-07-31T16:47:53.018Z+ zoneadm list
7322026-07-31T16:47:53.018Z+ grep nexus
7332026-07-31T16:47:53.018Z+ [[ 33 -gt 300 ]]
7342026-07-31T16:47:53.018Z+ sleep 1
7352026-07-31T16:47:54.010Z+ retry=34
7362026-07-31T16:47:54.034Z+ zoneadm list
7372026-07-31T16:47:54.034Z+ grep nexus
7382026-07-31T16:47:54.034Z+ [[ 34 -gt 300 ]]
7392026-07-31T16:47:54.034Z+ sleep 1
7402026-07-31T16:47:55.029Z+ retry=35
7412026-07-31T16:47:55.053Z+ zoneadm list
7422026-07-31T16:47:55.053Z+ grep nexus
7432026-07-31T16:47:55.053Z+ [[ 35 -gt 300 ]]
7442026-07-31T16:47:55.053Z+ sleep 1
7452026-07-31T16:47:56.045Z+ retry=36
7462026-07-31T16:47:56.069Z+ zoneadm list
7472026-07-31T16:47:56.069Z+ grep nexus
7482026-07-31T16:47:56.069Z+ [[ 36 -gt 300 ]]
7492026-07-31T16:47:56.069Z+ sleep 1
7502026-07-31T16:47:57.060Z+ retry=37
7512026-07-31T16:47:57.084Z+ zoneadm list
7522026-07-31T16:47:57.085Z+ grep nexus
7532026-07-31T16:47:57.085Z+ [[ 37 -gt 300 ]]
7542026-07-31T16:47:57.085Z+ sleep 1
7552026-07-31T16:47:58.082Z+ retry=38
7562026-07-31T16:47:59.080Z+ zoneadm list
7572026-07-31T16:47:59.109Z+ grep nexus
7582026-07-31T16:47:59.219Z+ [[ 38 -gt 300 ]]
7592026-07-31T16:47:59.243Z+ sleep 1
 
7602026-07-31T16:49:30.562Zsock console login:
7612026-07-31T16:49:30.621Zpanic[cpu0]/thread=fffffe00f3cf2c20: pageout_deadman: stuck pushing the same page for 90 seconds (freemem is 120238)
7622026-07-31T16:49:30.668Zfffffe00f3cf2a60 genunix:pageout_deadman+62 ()
7632026-07-31T16:49:30.713Zfffffe00f3cf2ad0 genunix:clock+7b3 ()
7642026-07-31T16:49:30.758Zfffffe00f3cf2b60 genunix:cyclic_softint+e1 ()
7652026-07-31T16:49:30.802Zfffffe00f3cf2b80 unix:cbe_softclock+23 ()
7662026-07-31T16:49:30.848Zfffffe00f3cf2bd0 unix:av_dispatch_softvect+72 ()
7672026-07-31T16:49:30.926Zfffffe00f3cf2c00 apix:apix_dispatch_softint+35 ()
7682026-07-31T16:49:30.967Zfffffe00f3c05970 unix:switch_sp_and_call+15 ()
7692026-07-31T16:49:31.012Zfffffe00f3c059c0 apix:apix_do_softint+5a ()
7702026-07-31T16:49:31.466Zfffffe00f3c05a20 apix:apix_do_interrupt+2bf ()
7712026-07-31T16:49:31.770Zfffffe00f3c05a30 unix:cmnint+c3 ()
7722026-07-31T16:49:31.872Zfffffe00f3c05ba0 unix:acpi_cpu_cstate+133 ()
7732026-07-31T16:49:31.916Zfffffe00f3c05bd0 unix:cpu_acpi_idle+75 ()
7742026-07-31T16:49:31.949Zfffffe00f3c05be0 unix:cpu_idle_adaptive+19 ()
7752026-07-31T16:49:31.983Zfffffe00f3c05c00 unix:idle+a8 ()
7762026-07-31T16:49:32.017Zfffffe00f3c05c10 unix:thread_start+b ()
7772026-07-31T16:49:32.050Zskipping system dump - no dump device configured
7782026-07-31T16:49:32.084Zrebooting...
7792026-07-31T16:51:05.263Z^[ciPXE initialising devices...ok
7802026-07-31T16:51:05.293Z^[[0m
7812026-07-31T16:51:05.394Z^[[1miPXE 1.21.1+ (g8bff5)^[[0m -- Open Source Network Boot Firmwar
7822026-07-31T16:51:05.426Z-- ^[[36mhttp://ipxe.org^[[0m
7832026-07-31T16:51:05.762ZFeatures: DNS HTTP iSCSI TFTP SRP AoE ELF MBOOT PXE bzImage Menu PXEXT
7842026-07-31T16:51:07.870Znet0: 18:c0:4d:0d:a0:2a using i350 on 0000:c3:00.0 (Ethernet) [open]
7852026-07-31T16:51:07.916Z[Link:up, TX:0 TXE:1 RX:0 RXE:0]
7862026-07-31T16:51:07.950Z[TXE: 1 x "Network unreachable (http://ipxe.org/28086011)"]
7872026-07-31T16:51:10.898ZConfiguring (net0 18:c0:4d:0d:a0:2a)...... ok
7882026-07-31T16:51:10.947Znet0: 10.151.2.100/255.255.255.0 gw 10.151.2.1
7892026-07-31T16:51:10.990ZFilename: http://172.20.0.5:9969/ipxe/sock
7902026-07-31T16:51:11.237Zhttp://172.20.0.5:9969/ipxe/sock... ok
7912026-07-31T16:51:11.520Zsock : 555 bytes [script]
7922026-07-31T16:53:06.646Zworker failed without completing job
7932026-07-31T16:53:06.648Ztask 2 was incomplete, marked failed